diff --git a/README.md b/README.md
index cf18493e..6e5efa5d 100644
--- a/README.md
+++ b/README.md
@@ -65,28 +65,59 @@ verification, pinning and how releases are cut.
Prerequisites: a cluster with [Agent Substrate](https://github.com/agent-substrate/substrate)
installed and a snapshots bucket. ate-env is built against Substrate
-[`3a2ca623`](https://github.com/agent-substrate/substrate/commit/3a2ca623) (2026-10-16, Go module
-`go.ate.dev/substrate`). The cluster needs a Substrate recent enough to accept what ate-env writes:
-the WorkerPool declares `spec.sandboxClasses`, the ActorTemplate uses `snapshotConfig` and
-`wakeupProbe`, names its sandbox class as the string `gvisor`, and declares CPU and memory limits.
-That means Substrate at or after
-[`63a17a92`](https://github.com/agent-substrate/substrate/commit/63a17a92) (2026-10-15), which also
-renamed the worker image: `--worker-image` must be the `ateworker-gvisor` image published by that
-Substrate install. An older Substrate rejects the string sandbox class (from
-[`9ec49759`](https://github.com/agent-substrate/substrate/commit/9ec49759)) or the limits (from
-[`c58f374a`](https://github.com/agent-substrate/substrate/commit/c58f374a)); use an ate-env from before
-this bump with it.
-
-When you upgrade an existing cluster across that boundary, register your base templates again.
-Substrate cannot update an ActorTemplate in place, and a base registered earlier has no CPU and memory
-limits, thus each template `ate-env-api` derives from it for a rootfs image fails with
-`resources: Required`. Delete the base and its derived templates (named `-<12 hex digits>`) with
-`kubectl-ate delete actor-template`, then register the base again with `ate-env manifest template`.
-Each environment created after that derives its template again, with the limits.
+[`ddb46651`](https://github.com/agent-substrate/substrate/commit/ddb46651) (2026-10-18, Go module
+`go.ate.dev/substrate`). The cluster needs Substrate at or after
+[`0443aec3`](https://github.com/agent-substrate/substrate/commit/0443aec3) (2026-10-17): from that
+commit the WorkerPool alone chooses the sandbox (`spec.sandboxClasses`), the ActorTemplate names
+none, and Substrate rejects a template that still sends `sandboxConfig`, while an older Substrate
+requires one. Use an ate-env from before this bump with an older Substrate. Because the template no
+longer names a class, an actor booting without a snapshot can land on any pool whose labels its
+template's `workerSelector` matches, and on any pool at all without a selector. `ate-env` labels its
+WorkerPool `ate-env/workerpool: ` and each template selects that label, thus actors stay on
+their pool and its sandbox class; `--workerpool` on `ate-env manifest template` must name the same
+pool as on `ate-env manifest` (both default to `-workerpool`). The templates also use
+`snapshotConfig` and `wakeupProbe` and declare CPU and memory limits, and `--worker-image` must be the
+`ateworker-gvisor` image published by the same Substrate install.
+
+`ate-env-api` suspends synchronously: a suspend returns once the snapshot is in object storage, thus a
+suspended environment survives the loss of its node. Substrate's own default became asynchronous in
+[`b40d34f1`](https://github.com/agent-substrate/substrate/commit/b40d34f1).
+
+Substrate enforces authorization by default. `ate-env-api` runs as its own ServiceAccount
+(`ate-env/ate-env-api`) and needs the editor role on each atespace it creates environments in; the
+atespace must exist first, because creating one takes a global owner. Step 1 sets both up.
+
+When you upgrade an existing cluster across these changes:
+
+- Create the atespace and the access policy (step 1) before you apply the new manifest. Without them
+ `ate-env-api` fails at startup with `PermissionDenied` while it checks the default atespace.
+- Base templates registered before keep working, because Substrate drops their old `sandboxConfig`
+ when it stores them, but they have no `workerSelector`. On a cluster with more than one
+ WorkerPool, register them again (delete the base and its `-<12 hex digits>` derived
+ templates, then `ate-env manifest template`), or their actors can boot on another pool. Apply the
+ new manifest first, so the pool carries the label the templates select.
+- A base registered before the CPU and memory limits (Substrate before
+ [`c58f374a`](https://github.com/agent-substrate/substrate/commit/c58f374a)) still needs registering
+ again: Substrate cannot update an ActorTemplate in place, thus each template `ate-env-api` derives
+ from it fails with `resources: Required`. Delete the base and its derived templates (named
+ `-<12 hex digits>`) with `kubectl-ate delete actor-template`, then register the base again.
### 1. Deploy the Kubernetes Resources
-Deploy the namespace, worker pool, and API service:
+Give `ate-env-api` access to its atespace, once, as an owner of the atespace (on a stock install the
+default `kubectl-ate` identity is a bootstrap owner):
+
+```bash
+kubectl ate create atespace ate-env
+ate-env manifest access-policy | kubectl ate create access-policy -a ate-env -f -
+```
+
+An atespace has one access policy. If `ate-env` already has one, add the binding that
+`ate-env manifest access-policy` prints to it with `kubectl ate get access-policy` and
+`kubectl ate update access-policy`. Repeat both commands for each other atespace you create
+environments in.
+
+Deploy the namespace, worker pool, ServiceAccount and API service:
```bash
# With a release binary the API image defaults to the release's. The worker
diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go
index 9552aec2..1dc45be0 100644
--- a/cmd/ate-env/manifest.go
+++ b/cmd/ate-env/manifest.go
@@ -38,9 +38,21 @@ import (
// apiName is the name of the API service Deployment and Service.
const apiName = "ate-env-api"
-// sandboxConfigName is the Substrate SandboxConfig both the WorkerPool and the
-// ActorTemplate refer to. `ate-setup deploy ate-system` installs exactly this
-// one, so a pool declared with it is schedulable on a stock install.
+// workerPoolLabel labels the WorkerPool with its own name, and templates
+// select their pool by it. Substrate copies a pool's labels to its workers and
+// matches a template's workerSelector against them. Since substrate#2780 the
+// template names no sandbox class, thus without a selector an actor that
+// boots without a snapshot can land on any pool of any class in the cluster.
+const workerPoolLabel = "ate-env/workerpool"
+
+// defaultWorkerPool is the pool name both commands use unless --workerpool
+// says otherwise.
+func defaultWorkerPool(template string) string { return template + "-workerpool" }
+
+// sandboxConfigName is the Substrate SandboxConfig the WorkerPool's gvisor
+// sandbox class refers to. `ate-setup deploy ate-system` installs exactly this
+// one, so a pool declared with it is schedulable on a stock install. Since
+// substrate#2780 the ActorTemplate names no sandbox: the worker pool decides.
const sandboxConfigName = "gvisor-default"
// Default compute limits for each actor. Substrate requires both on every
@@ -84,7 +96,10 @@ binaries default it to the ate-env-api image published with the release
// templateConfig holds configuration for generating a Substrate ActorTemplate manifest.
type templateConfig struct {
- template string
+ template string
+ // workerPool is the WorkerPool the template's actors run on; the template
+ // selects it by workerPoolLabel.
+ workerPool string
atespace string
guestImage string
guestCommand []string
@@ -244,7 +259,7 @@ the "template" subcommand: ate-env manifest template`,
mCfg.template = apiservice.DefaultTemplate
}
if mCfg.workerPool == "" {
- mCfg.workerPool = mCfg.template + "-workerpool"
+ mCfg.workerPool = defaultWorkerPool(mCfg.template)
}
if err := mCfg.resolveImages(); err != nil {
return err
@@ -264,10 +279,38 @@ the "template" subcommand: ate-env manifest template`,
cmd.Flags().Int32Var(&mCfg.replicas, "replicas", 5, "number of pre-warmed worker pods")
cmd.AddCommand(newManifestTemplateCommand())
+ cmd.AddCommand(newManifestAccessPolicyCommand())
return cmd
}
+func newManifestAccessPolicyCommand() *cobra.Command {
+ namespace := apiservice.DefaultNamespace
+ cmd := &cobra.Command{
+ Use: "access-policy",
+ Short: "Generate the Substrate access policy that lets ate-env-api manage environments",
+ Long: `Access-policy prints a Substrate access policy, in the form kubectl-ate reads,
+that grants ate-env-api's ServiceAccount the editor role on an atespace.
+
+Substrate enforces authorization by default: ate-env-api then needs editor on
+each atespace it creates environments in, and the atespace must exist, because
+creating one takes a global owner. As an owner of the atespace (the default
+kubectl-ate identity on a stock install), run once for each atespace:
+
+ kubectl ate create atespace ate-env
+ ate-env manifest access-policy | kubectl ate create access-policy -a ate-env -f -
+
+An atespace has one policy. If it already has one, add the binding to it with
+"kubectl ate get access-policy" and "kubectl ate update access-policy".`,
+ Args: cobra.NoArgs,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return writeProtoYAML(cmd.OutOrStdout(), buildAccessPolicy(namespace), "access policy")
+ },
+ }
+ cmd.Flags().StringVar(&namespace, "namespace", namespace, "Kubernetes namespace ate-env-api runs in (its ServiceAccount's namespace)")
+ return cmd
+}
+
func newManifestTemplateCommand() *cobra.Command {
tCfg := templateConfig{}
@@ -281,6 +324,9 @@ It prints YAML to stdout without touching the cluster.`,
if tCfg.template == "" {
tCfg.template = apiservice.DefaultTemplate
}
+ if tCfg.workerPool == "" {
+ tCfg.workerPool = defaultWorkerPool(tCfg.template)
+ }
if err := tCfg.resolveImages(); err != nil {
return err
}
@@ -302,6 +348,7 @@ It prints YAML to stdout without touching the cluster.`,
}
cmd.Flags().StringVar(&tCfg.template, "template", apiservice.DefaultTemplate, "ActorTemplate name")
+ cmd.Flags().StringVar(&tCfg.workerPool, "workerpool", "", "WorkerPool the actors run on, selected by its "+workerPoolLabel+" label (defaults to -workerpool, as in ate-env manifest)")
cmd.Flags().StringVar(&tCfg.atespace, "atespace", apiservice.DefaultAtespace, "Substrate atespace for the ActorTemplate")
cmd.Flags().StringVar(&tCfg.guestImage, "guest-image", defaultGuestImage, imageDefaultHelp("digest-pinned ate-env-guest image (repo@sha256:...)", defaultGuestImage))
cmd.Flags().StringSliceVar(&tCfg.guestCommand, "guest-command", []string{"/ko-app/ate-env-guest"}, "guest container entrypoint")
@@ -323,29 +370,73 @@ func buildManifests(cfg manifestConfig) []any {
return []any{
buildNamespace(cfg),
buildWorkerPool(cfg),
+ buildAPIServiceAccount(cfg),
buildAPIDeployment(cfg),
buildAPIService(cfg),
}
}
+// apiPrincipal is the identity ate-env-api presents to the Substrate control
+// plane: its own ServiceAccount, through a projected token. Access policies
+// name it with a "user:" prefix (see the access-policy subcommand).
+func apiPrincipal(namespace string) string {
+ return "system:serviceaccount:" + namespace + ":" + apiName
+}
+
+// buildAPIServiceAccount returns the ServiceAccount ate-env-api runs as. A
+// dedicated account gives the control plane a stable principal to authorize;
+// the namespace's default account would be shared with anything else there.
+// ate-env-api never calls the Kubernetes API, thus no API token is mounted.
+func buildAPIServiceAccount(cfg manifestConfig) *corev1.ServiceAccount {
+ automount := false
+ return &corev1.ServiceAccount{
+ TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ServiceAccount"},
+ ObjectMeta: metav1.ObjectMeta{
+ Name: apiName,
+ Namespace: cfg.namespace,
+ Labels: map[string]string{"app": apiName},
+ },
+ AutomountServiceAccountToken: &automount,
+ }
+}
+
+// buildAccessPolicy returns the Substrate access policy that lets ate-env-api
+// manage environments in one atespace: the editor role creates, suspends and
+// deletes actors, and creates the per-image templates it derives.
+func buildAccessPolicy(namespace string) *ateapipb.AccessPolicy {
+ return &ateapipb.AccessPolicy{
+ Metadata: &ateapipb.ResourceMetadata{Name: "default"},
+ Bindings: []*ateapipb.Binding{{
+ Role: "editor",
+ Members: []string{"user:" + apiPrincipal(namespace)},
+ }},
+ }
+}
+
// writeActorTemplate writes a single Substrate ActorTemplate as protojson-shaped YAML.
func writeActorTemplate(w io.Writer, tmpl *ateapipb.ActorTemplate) error {
+ return writeProtoYAML(w, tmpl, "actor template")
+}
+
+// writeProtoYAML writes a Substrate control-plane message as protojson-shaped
+// YAML, the form kubectl-ate reads, without its server-managed status.
+func writeProtoYAML(w io.Writer, msg proto.Message, what string) error {
opts := protojson.MarshalOptions{
UseProtoNames: false,
}
- jsonBytes, err := opts.Marshal(tmpl)
+ jsonBytes, err := opts.Marshal(msg)
if err != nil {
- return fmt.Errorf("encoding actor template json: %w", err)
+ return fmt.Errorf("encoding %s json: %w", what, err)
}
var m map[string]any
if err := json.Unmarshal(jsonBytes, &m); err != nil {
- return fmt.Errorf("decoding actor template json: %w", err)
+ return fmt.Errorf("decoding %s json: %w", what, err)
}
delete(m, "status")
prune(m)
data, err := yaml.Marshal(m)
if err != nil {
- return fmt.Errorf("encoding actor template yaml: %w", err)
+ return fmt.Errorf("encoding %s yaml: %w", what, err)
}
_, err = w.Write(data)
return err
@@ -440,17 +531,19 @@ func buildWorkerPool(cfg manifestConfig) *atev1alpha1.WorkerPool {
ObjectMeta: metav1.ObjectMeta{
Name: cfg.workerPool,
Namespace: cfg.namespace,
+ // Copied to the pool's workers; templates select the pool by it.
+ Labels: map[string]string{workerPoolLabel: cfg.workerPool},
},
Spec: atev1alpha1.WorkerPoolSpec{
Replicas: cfg.replicas,
WorkerImage: cfg.workerImage,
- // The CRD requires exactly one sandbox class per pool, and
- // actors only schedule onto workers of their template's class,
- // so keep this in step with buildActorTemplate. The configRef is
- // the SandboxConfig the workers run; the template names the same
- // one in sandboxConfig.configName.
+ // The CRD requires exactly one sandbox class per pool. Since
+ // substrate#2780 the pool alone decides the sandbox: the
+ // ActorTemplate names none, and an actor restores onto a worker of
+ // its snapshot's class. The configRef is the SandboxConfig the
+ // workers run.
SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{
- Name: atev1alpha1.SandboxClassGvisor, // the template's sandboxConfig.sandboxClass is the same string
+ Name: atev1alpha1.SandboxClassGvisor,
ConfigRef: &atev1alpha1.SandboxConfigReference{Name: sandboxConfigName},
}},
},
@@ -462,11 +555,18 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate {
if atespace == "" {
atespace = apiservice.DefaultAtespace
}
+ pool := cfg.workerPool
+ if pool == "" {
+ pool = defaultWorkerPool(cfg.template)
+ }
return &ateapipb.ActorTemplate{
Metadata: &ateapipb.ResourceMetadata{
Name: cfg.template,
Atespace: atespace,
},
+ // Pin the actors to their pool, thus to its sandbox class: the
+ // template names no class since substrate#2780.
+ WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{workerPoolLabel: pool}},
Containers: []*ateapipb.Container{{
Name: "guest",
Image: cfg.guestImage,
@@ -489,10 +589,6 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate {
// schema marks the field required, so say it explicitly.
PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY,
},
- SandboxConfig: &ateapipb.SandboxConfig{
- SandboxClass: string(atev1alpha1.SandboxClassGvisor),
- ConfigName: sandboxConfigName,
- },
Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{
{Name: "cpu", Quantity: orDefault(cfg.cpu, defaultActorCPU)},
{Name: "memory", Quantity: orDefault(cfg.memory, defaultActorMemory)},
@@ -548,6 +644,7 @@ func buildTemplate(cfg templateConfig) (*ateapipb.ActorTemplate, error) {
// buildAPIDeployment returns the ate-env-api Deployment, pointed at the
// in-cluster Substrate endpoints.
func buildAPIDeployment(cfg manifestConfig) *appsv1.Deployment {
+ automountAPIToken := false
labels := map[string]string{"app": apiName}
replicas := cfg.apiReplicas
tokenExpiration := int64(7200)
@@ -564,6 +661,11 @@ func buildAPIDeployment(cfg manifestConfig) *appsv1.Deployment {
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels},
Spec: corev1.PodSpec{
+ // The ServiceAccount is the principal the control plane
+ // authorizes (buildAccessPolicy); the projected ateapi token
+ // below is minted for it.
+ ServiceAccountName: apiName,
+ AutomountServiceAccountToken: &automountAPIToken,
Containers: []corev1.Container{{
Name: apiName,
Image: cfg.apiImage,
diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go
index 95f26594..28507faf 100644
--- a/cmd/ate-env/manifest_test.go
+++ b/cmd/ate-env/manifest_test.go
@@ -24,6 +24,7 @@ import (
"go.ate.dev/substrate/pkg/proto/ateapipb"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
+ "sigs.k8s.io/yaml"
)
func testManifestConfig() manifestConfig {
@@ -84,8 +85,8 @@ func TestTemplateConfigResolveImages(t *testing.T) {
func TestBuildManifests(t *testing.T) {
cfg := testManifestConfig()
objs := buildManifests(cfg)
- if len(objs) != 4 {
- t.Fatalf("got %d manifests, want 4", len(objs))
+ if len(objs) != 5 {
+ t.Fatalf("got %d manifests, want 5", len(objs))
}
ns := objs[0].(*corev1.Namespace)
@@ -103,8 +104,20 @@ func TestBuildManifests(t *testing.T) {
if pool.Spec.Replicas != 3 || pool.Spec.WorkerImage != cfg.workerImage {
t.Errorf("workerpool spec = %+v, want replicas 3 and worker image %q", pool.Spec, cfg.workerImage)
}
- if len(pool.Labels) != 0 {
- t.Errorf("workerpool labels = %v, want empty", pool.Labels)
+ if got := pool.Labels; len(got) != 1 || got[workerPoolLabel] != pool.Name {
+ t.Errorf("workerpool labels = %v, want %s=%s", got, workerPoolLabel, pool.Name)
+ }
+ // The default template selects exactly this pool: Substrate copies the
+ // pool's labels to its workers and matches the template's selector
+ // against them, an AND over equal labels.
+ sel := buildActorTemplate(testTemplateConfig()).GetWorkerSelector().GetMatchLabels()
+ for k, v := range sel {
+ if pool.Labels[k] != v {
+ t.Errorf("template selector %v does not match pool labels %v", sel, pool.Labels)
+ }
+ }
+ if len(sel) == 0 {
+ t.Error("template has no worker selector; it can cold-boot on any pool")
}
if pool.Spec.Template != nil {
t.Errorf("workerpool template = %v, want nil", pool.Spec.Template)
@@ -116,15 +129,15 @@ func TestBuildManifests(t *testing.T) {
if sc.Name != atev1alpha1.SandboxClassGvisor || sc.ConfigRef == nil || sc.ConfigRef.Name != sandboxConfigName {
t.Errorf("workerpool sandboxClasses[0] = %+v, want gvisor with configRef %q", sc, sandboxConfigName)
}
- // The pool's class and the template's are the same string since
- // substrate#2586 made the proto field a string. A pool of the wrong class
- // never hosts the template's actors.
- tmplClass := buildActorTemplate(testTemplateConfig()).GetSandboxConfig().GetSandboxClass()
- if tmplClass != string(sc.Name) {
- t.Errorf("workerpool sandbox class %q != template sandbox class %q", sc.Name, tmplClass)
+ sa := objs[2].(*corev1.ServiceAccount)
+ if sa.Name != apiName || sa.Namespace != cfg.namespace || sa.AutomountServiceAccountToken == nil || *sa.AutomountServiceAccountToken {
+ t.Errorf("service account = %s/%s automount %v, want %s/%s with no API token", sa.Namespace, sa.Name, sa.AutomountServiceAccountToken, cfg.namespace, apiName)
}
- deployment := objs[2].(*appsv1.Deployment)
+ deployment := objs[3].(*appsv1.Deployment)
+ if pod := deployment.Spec.Template.Spec; pod.ServiceAccountName != apiName {
+ t.Errorf("api pod service account = %q, want %q (the principal the access policy names)", pod.ServiceAccountName, apiName)
+ }
containers := deployment.Spec.Template.Spec.Containers
if len(containers) != 1 || containers[0].Image != cfg.apiImage {
t.Errorf("api containers = %+v, want one container with image %q", containers, cfg.apiImage)
@@ -133,7 +146,7 @@ func TestBuildManifests(t *testing.T) {
t.Errorf("api replicas = %d, want 1", *deployment.Spec.Replicas)
}
- service := objs[3].(*corev1.Service)
+ service := objs[4].(*corev1.Service)
if service.Spec.Selector["app"] != apiName {
t.Errorf("api service selector = %v, want app=%s", service.Spec.Selector, apiName)
}
@@ -165,11 +178,8 @@ func TestBuildActorTemplate(t *testing.T) {
if got := template.GetSnapshotConfig().GetGoldenSnapshotConfig(); got != nil {
t.Errorf("golden snapshot config = %v, want nil (omitted means enabled)", got)
}
- if got := template.GetSandboxConfig().GetConfigName(); got != sandboxConfigName {
- t.Errorf("sandbox config name = %q, want %q (must match the WorkerPool configRef)", got, sandboxConfigName)
- }
- if template.GetWorkerSelector() != nil {
- t.Errorf("worker selector = %v, want nil", template.WorkerSelector)
+ if got := template.GetWorkerSelector().GetMatchLabels(); len(got) != 1 || got[workerPoolLabel] != "default-template-workerpool" {
+ t.Errorf("worker selector = %v, want %s=default-template-workerpool", got, workerPoolLabel)
}
probe := template.Containers[0].GetWakeupProbe()
if probe.GetHttpGet() == nil || probe.GetHttpGet().GetPath() != "/readyz" || probe.GetHttpGet().GetPort() != 80 {
@@ -179,9 +189,6 @@ func TestBuildActorTemplate(t *testing.T) {
if got := limits(template); got["cpu"] != defaultActorCPU || got["memory"] != defaultActorMemory || len(got) != 2 {
t.Errorf("resources.limits = %v, want cpu=%s memory=%s", got, defaultActorCPU, defaultActorMemory)
}
- if got := template.GetSandboxConfig().GetSandboxClass(); got != "gvisor" {
- t.Errorf("sandbox class = %q, want gvisor (Substrate accepts gvisor or microvm)", got)
- }
}
func limits(tmpl *ateapipb.ActorTemplate) map[string]string {
@@ -250,7 +257,7 @@ func TestBuildManifestsCustomAPIPort(t *testing.T) {
cfg.apiPort = 9999
objs := buildManifests(cfg)
- deployment := objs[2].(*appsv1.Deployment)
+ deployment := objs[3].(*appsv1.Deployment)
container := deployment.Spec.Template.Spec.Containers[0]
if got := container.Args[1]; got != "0.0.0.0:9999" {
t.Errorf("api -listen arg = %q, want 0.0.0.0:9999", got)
@@ -262,7 +269,7 @@ func TestBuildManifestsCustomAPIPort(t *testing.T) {
t.Errorf("api probe port = %d, want 9999", got)
}
- service := objs[3].(*corev1.Service)
+ service := objs[4].(*corev1.Service)
if service.Spec.Ports[0].Port != 9999 || service.Spec.Ports[0].TargetPort.IntValue() != 9999 {
t.Errorf("api service ports = %+v, want 9999 -> 9999", service.Spec.Ports)
}
@@ -275,13 +282,15 @@ func TestWriteManifests(t *testing.T) {
}
out := buf.String()
- if got := strings.Count(out, "\n---\n"); got != 3 {
- t.Errorf("got %d document separators, want 3:\n%s", got, out)
+ if got := strings.Count(out, "\n---\n"); got != 4 {
+ t.Errorf("got %d document separators, want 4:\n%s", got, out)
}
for _, want := range []string{
"kind: Namespace",
"ate.dev/api-access: \"true\"",
"kind: WorkerPool",
+ "kind: ServiceAccount",
+ "serviceAccountName: ate-env-api",
"kind: Deployment",
"kind: Service",
"sandboxClasses:",
@@ -368,11 +377,15 @@ func TestManifestTemplateCommandExecution(t *testing.T) {
if !strings.Contains(out, "image: example.com/guest@sha256:aaaa") || !strings.Contains(out, "storageLocation: gs://bucket/prefix/") {
t.Errorf("expected guest image and storage location in output:\n%s", out)
}
- for _, want := range []string{"sandboxClass: gvisor", "name: cpu\n quantity: \"4\"", "name: memory\n quantity: 8Gi"} {
+ for _, want := range []string{"name: cpu\n quantity: \"4\"", "name: memory\n quantity: 8Gi"} {
if !strings.Contains(out, want) {
t.Errorf("output lacks %q:\n%s", want, out)
}
}
+ // Substrate rejects a template with sandboxConfig since substrate#2780.
+ if strings.Contains(out, "sandboxConfig") || strings.Contains(out, "sandboxClass") {
+ t.Errorf("template names a sandbox; the worker pool decides it now:\n%s", out)
+ }
}
const testHex64 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
@@ -531,3 +544,58 @@ func TestImageFlagsDefaultToReleaseImages(t *testing.T) {
t.Errorf("missing --worker-image should point at the Substrate repo, got %v", err)
}
}
+
+func TestManifestAccessPolicyCommand(t *testing.T) {
+ for _, ns := range []string{"", "team-env"} {
+ cmd := newManifestCommand()
+ var out bytes.Buffer
+ cmd.SetOut(&out)
+ args := []string{"access-policy"}
+ if ns != "" {
+ args = append(args, "--namespace", ns)
+ }
+ cmd.SetArgs(args)
+ if err := cmd.Execute(); err != nil {
+ t.Fatal(err)
+ }
+ want := "user:system:serviceaccount:" + map[bool]string{true: "ate-env", false: ns}[ns == ""] + ":ate-env-api"
+ var m map[string]any
+ if err := yaml.Unmarshal(out.Bytes(), &m); err != nil {
+ t.Fatalf("%q: %v", out.String(), err)
+ }
+ md, _ := m["metadata"].(map[string]any)
+ bindings, _ := m["bindings"].([]any)
+ if md["name"] != "default" || md["atespace"] != nil || len(bindings) != 1 {
+ t.Fatalf("policy = %v, want metadata.name default, no atespace, one binding", m)
+ }
+ b := bindings[0].(map[string]any)
+ members, _ := b["members"].([]any)
+ if b["role"] != "editor" || len(members) != 1 || members[0] != want {
+ t.Errorf("binding = %v, want editor for %s", b, want)
+ }
+ }
+}
+
+func TestManifestTemplateSelectsTheNamedPool(t *testing.T) {
+ cmd := newManifestCommand()
+ var out bytes.Buffer
+ cmd.SetOut(&out)
+ cmd.SetArgs([]string{"template", "--template", "py", "--workerpool", "team-pool",
+ "--guest-image=example.com/guest@sha256:aaaa", "--snapshots-bucket=gs://b/p/"})
+ if err := cmd.Execute(); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(out.String(), "workerSelector:\n matchLabels:\n ate-env/workerpool: team-pool") {
+ t.Errorf("template does not select team-pool:\n%s", out.String())
+ }
+ out.Reset()
+ cmd = newManifestCommand()
+ cmd.SetOut(&out)
+ cmd.SetArgs([]string{"template", "--template", "py", "--guest-image=example.com/guest@sha256:aaaa", "--snapshots-bucket=gs://b/p/"})
+ if err := cmd.Execute(); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(out.String(), "ate-env/workerpool: py-workerpool") {
+ t.Errorf("default pool is not -workerpool:\n%s", out.String())
+ }
+}
diff --git a/docs/rootfs-images/DESIGN.md b/docs/rootfs-images/DESIGN.md
index fd9dfcf6..fe976752 100644
--- a/docs/rootfs-images/DESIGN.md
+++ b/docs/rootfs-images/DESIGN.md
@@ -146,8 +146,9 @@ piece, and it belongs with the OpenSandbox backend that needs it.
## Security
-- The rootfs image is run with the same sandbox class and configuration as the
- base, so injection changes what runs, not how it is isolated.
+- The rootfs image runs in the same sandbox as any other actor of the worker
+ pool (the pool's sandbox class decides it; the template names none), so
+ injection changes what runs, not how it is isolated.
- The guest volume is read-only. The rootfs image cannot alter the guest.
- Digest pinning is enforced for both images at every entry point. A tag could
be re-pointed between the golden bake and a later cold boot, which would
diff --git a/docs/rootfs-images/README.md b/docs/rootfs-images/README.md
index 2843f20c..a0cf9265 100644
--- a/docs/rootfs-images/README.md
+++ b/docs/rootfs-images/README.md
@@ -19,7 +19,7 @@ process = /ate/ko-app/ate-env-guest, from a second image mounted read-only at /
```
Substrate materializes both at actor start. Everything else about the template
-(worker selector, snapshot settings, sandbox class, resources, readiness probe)
+(worker selector, snapshot settings, resources, readiness probe)
is the same as for a guest-image template, and the guest behaves identically:
the environment's `shell`, process and file APIs, MCP, and suspend and resume
all work unchanged.
@@ -63,7 +63,7 @@ ate-env create job1 --template py312
`--rootfs-image` makes the rootfs image the container and the guest an image
volume; `--workspace` is passed to the guest as `-workspace` in either mode. The
printed template is plain YAML you can edit before registering it (resources,
-worker selector, sandbox class).
+worker selector). The sandbox class is the worker pool's, not the template's.
## Option B: create from an image on demand
@@ -137,7 +137,7 @@ runnable example in [`examples/runtime-layers`](../../examples/runtime-layers).
- Bumping the guest means a new base template (templates are immutable), and a
new base name yields new derived names. Existing derived templates keep the
guest they were built with.
-- Resources, sandbox class and worker selector come from the base; a rootfs image
+- Resources and worker selector come from the base, and the sandbox from the worker pool; a rootfs image
that needs more is served by registering its own template (Option A) or a
second base.
- The rootfs image's existence is not checked at create time. A bad reference is
diff --git a/go.mod b/go.mod
index f611624d..a911b31c 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
github.com/google/uuid v1.6.0
github.com/modelcontextprotocol/go-sdk v1.8.0
github.com/spf13/cobra v1.10.2
- go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748
+ go.ate.dev/substrate v0.5.1-0.20261018212731-ddb46651d605
google.golang.org/grpc v1.84.0
google.golang.org/protobuf v1.36.12
k8s.io/api v0.37.1
diff --git a/go.sum b/go.sum
index edf211fe..166d15cd 100644
--- a/go.sum
+++ b/go.sum
@@ -109,8 +109,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
-go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748 h1:QrTauJhnfn0FL0t/GLzVo4bmeVUYMT6e4x8QSaQF8s0=
-go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748/go.mod h1:EktaeScxg724tF4cjtLV8kdz2xN3FH8NOTk/A95jSBY=
+go.ate.dev/substrate v0.5.1-0.20261018212731-ddb46651d605 h1:0hbCfZd53YPGYhtBM0mHs/UCz66J47l0GTeMscaIezk=
+go.ate.dev/substrate v0.5.1-0.20261018212731-ddb46651d605/go.mod h1:EktaeScxg724tF4cjtLV8kdz2xN3FH8NOTk/A95jSBY=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go
index d7050e23..fc5d831b 100644
--- a/internal/apiservice/imagetemplate_test.go
+++ b/internal/apiservice/imagetemplate_test.go
@@ -39,7 +39,6 @@ func modeABase() *ateapipb.ActorTemplate {
WakeupProbe: &ateapipb.WakeupProbe{HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}},
}},
SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://b/p/", PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY},
- SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"},
Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{
{Name: "cpu", Quantity: "2"}, {Name: "memory", Quantity: "4Gi"},
}},
@@ -124,10 +123,9 @@ func TestDeriveImageTemplateFromGuestImageBase(t *testing.T) {
// Everything else carries over.
if !proto.Equal(got.GetWorkerSelector(), base.GetWorkerSelector()) ||
!proto.Equal(got.GetSnapshotConfig(), base.GetSnapshotConfig()) ||
- !proto.Equal(got.GetSandboxConfig(), base.GetSandboxConfig()) ||
!proto.Equal(c.GetWakeupProbe(), base.GetContainers()[0].GetWakeupProbe()) ||
len(c.GetEnv()) != 1 {
- t.Error("worker selector, snapshots, sandbox config, readyz or env did not carry over")
+ t.Error("worker selector, snapshots, readyz or env did not carry over")
}
}
diff --git a/internal/apiservice/server_test.go b/internal/apiservice/server_test.go
index 9ecccd5c..a9dfae2d 100644
--- a/internal/apiservice/server_test.go
+++ b/internal/apiservice/server_test.go
@@ -599,7 +599,6 @@ func seedGuestTemplate(control *fakecontrol.Server, atespace, name string) {
WakeupProbe: &ateapipb.WakeupProbe{HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}},
}},
SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://bucket/ate-env/"},
- SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"},
})
}
diff --git a/internal/ate/client.go b/internal/ate/client.go
index 344e953e..b21d88ba 100644
--- a/internal/ate/client.go
+++ b/internal/ate/client.go
@@ -296,8 +296,18 @@ func (c *Client) Get(ctx context.Context, atespace, id string) (*ateapipb.Actor,
// Suspend checkpoints and stops the actor with ID in atespace.
// If atespace is empty, DefaultAtespace is used.
+//
+// The suspend is synchronous: it returns once the snapshot is in durable
+// storage, thus a suspended environment survives the loss of its node.
+// Substrate's default became asynchronous in substrate#2805 (the snapshot
+// stays on the node and uploads in the background); ate-env keeps the
+// durable behavior it always had.
func (c *Client) Suspend(ctx context.Context, atespace, id string) error {
- if _, err := c.control.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: c.ref(atespace, id)}); err != nil {
+ req := &ateapipb.SuspendActorRequest{
+ Actor: c.ref(atespace, id),
+ Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS},
+ }
+ if _, err := c.control.SuspendActor(ctx, req); err != nil {
return fmt.Errorf("ate: suspending %q: %w", id, wrapGRPCError(err))
}
return nil
diff --git a/internal/ate/client_test.go b/internal/ate/client_test.go
index 727e5fcf..e2786aa1 100644
--- a/internal/ate/client_test.go
+++ b/internal/ate/client_test.go
@@ -118,6 +118,11 @@ func TestSuspend(t *testing.T) {
if st := f.control.Status("sb-susp"); st != ateapipb.ActorState_ACTOR_STATE_SUSPENDED {
t.Fatalf("status = %v, want SUSPENDED", st)
}
+ // Substrate's default is asynchronous since substrate#2805; ate-env asks
+ // for a durable suspend.
+ if got := f.control.SuspendModes; len(got) != 1 || got[0] != ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS {
+ t.Fatalf("suspend modes = %v, want one SYNCHRONOUS", got)
+ }
}
func TestSuspendMissing(t *testing.T) {
diff --git a/internal/internaltest/fakecontrol/fakecontrol.go b/internal/internaltest/fakecontrol/fakecontrol.go
index 1a87acc1..0adf3fba 100644
--- a/internal/internaltest/fakecontrol/fakecontrol.go
+++ b/internal/internaltest/fakecontrol/fakecontrol.go
@@ -51,6 +51,8 @@ type Server struct {
// Substrate does for an identity allowed to use but not create atespaces.
DenyCreateAtespace bool
templates map[string]*ateapipb.ActorTemplate
+ // SuspendModes records the durability mode of each SuspendActor call.
+ SuspendModes []ateapipb.DurabilityMode
}
// New returns an empty fake control server.
@@ -272,6 +274,7 @@ func (s *Server) SuspendActor(ctx context.Context, req *ateapipb.SuspendActorReq
if err != nil {
return nil, err
}
+ s.SuspendModes = append(s.SuspendModes, req.GetOptions().GetMode())
s.suspend(a)
return &ateapipb.SuspendActorResponse{Actor: clone(a)}, nil
}
@@ -292,18 +295,6 @@ func (s *Server) suspend(a *ateapipb.Actor) {
}
}
-func (s *Server) PauseActor(ctx context.Context, req *ateapipb.PauseActorRequest) (*ateapipb.PauseActorResponse, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- a, err := s.get(req.GetActor())
- if err != nil {
- return nil, err
- }
- // Substrate reports a paused actor as SUSPENDED (substrate#2620).
- a.Status = &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}
- return &ateapipb.PauseActorResponse{Actor: clone(a)}, nil
-}
-
func (s *Server) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) {
s.mu.Lock()
defer s.mu.Unlock()