From 0ce50ef0f3ecc2c0abeb0007b4324944cd51e331 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:30:18 -0700 Subject: [PATCH 1/3] Bump Substrate to ddb46651: templates name no sandbox, durable suspend, an identity Substrate's authorization can grant go.ate.dev/substrate 3a2ca623 -> ddb46651 (2026-10-18). - substrate#2780 removed ActorTemplate.sandbox_config and the SandboxConfig message; the WorkerPool's sandboxClasses alone chooses the sandbox, and ate-api-server rejects a template that still sends sandboxConfig. The generated template drops it. The WorkerPool already declares gvisor with the gvisor-default configRef. An older Substrate requires the field, thus this ate-env needs Substrate at or after 0443aec3. - substrate#2813 removed PauseActor; the fake control plane drops it. - substrate#2805 made SuspendActor asynchronous by default (the snapshot stays on the node and uploads in the background). ate-env-api asks for DURABILITY_MODE_SYNCHRONOUS, the behavior it always had, so a suspended environment survives the loss of its node. The fake records the mode and TestSuspend checks it. - Substrate enforces ateapi authorization by default (substrate#2591; also in the previous pin). ate-env-api ran as the namespace's default ServiceAccount, which no policy grants, so each control-plane call was denied on a stock install. `ate-env manifest` now creates an ate-env-api ServiceAccount (no Kubernetes API token mounted; ate-env-api never calls the API) and runs the Deployment as it, and `ate-env manifest access-policy` prints the policy that grants it editor on an atespace, for `kubectl ate create access-policy -a -f -`. --- cmd/ate-env/manifest.go | 109 +++++++++++++++--- cmd/ate-env/manifest_test.go | 74 ++++++++---- go.mod | 2 +- go.sum | 4 +- internal/apiservice/imagetemplate_test.go | 4 +- internal/apiservice/server_test.go | 1 - internal/ate/client.go | 12 +- internal/ate/client_test.go | 5 + .../internaltest/fakecontrol/fakecontrol.go | 15 +-- 9 files changed, 168 insertions(+), 58 deletions(-) diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index 9552aec2..fdb8093f 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -38,9 +38,10 @@ import ( // apiName is the name of the API service Deployment and Service. const apiName = "ate-env-api" -// sandboxConfigName is the Substrate SandboxConfig both the WorkerPool and the -// ActorTemplate refer to. `ate-setup deploy ate-system` installs exactly this -// one, so a pool declared with it is schedulable on a stock install. +// sandboxConfigName is the Substrate SandboxConfig the WorkerPool's gvisor +// sandbox class refers to. `ate-setup deploy ate-system` installs exactly this +// one, so a pool declared with it is schedulable on a stock install. Since +// substrate#2780 the ActorTemplate names no sandbox: the worker pool decides. const sandboxConfigName = "gvisor-default" // Default compute limits for each actor. Substrate requires both on every @@ -264,10 +265,38 @@ the "template" subcommand: ate-env manifest template`, cmd.Flags().Int32Var(&mCfg.replicas, "replicas", 5, "number of pre-warmed worker pods") cmd.AddCommand(newManifestTemplateCommand()) + cmd.AddCommand(newManifestAccessPolicyCommand()) return cmd } +func newManifestAccessPolicyCommand() *cobra.Command { + namespace := apiservice.DefaultNamespace + cmd := &cobra.Command{ + Use: "access-policy", + Short: "Generate the Substrate access policy that lets ate-env-api manage environments", + Long: `Access-policy prints a Substrate access policy, in the form kubectl-ate reads, +that grants ate-env-api's ServiceAccount the editor role on an atespace. + +Substrate enforces authorization by default: ate-env-api then needs editor on +each atespace it creates environments in, and the atespace must exist, because +creating one takes a global owner. As an owner of the atespace (the default +kubectl-ate identity on a stock install), run once for each atespace: + + kubectl ate create atespace ate-env + ate-env manifest access-policy | kubectl ate create access-policy -a ate-env -f - + +An atespace has one policy. If it already has one, add the binding to it with +"kubectl ate get access-policy" and "kubectl ate update access-policy".`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return writeProtoYAML(cmd.OutOrStdout(), buildAccessPolicy(namespace), "access policy") + }, + } + cmd.Flags().StringVar(&namespace, "namespace", namespace, "Kubernetes namespace ate-env-api runs in (its ServiceAccount's namespace)") + return cmd +} + func newManifestTemplateCommand() *cobra.Command { tCfg := templateConfig{} @@ -323,29 +352,73 @@ func buildManifests(cfg manifestConfig) []any { return []any{ buildNamespace(cfg), buildWorkerPool(cfg), + buildAPIServiceAccount(cfg), buildAPIDeployment(cfg), buildAPIService(cfg), } } +// apiPrincipal is the identity ate-env-api presents to the Substrate control +// plane: its own ServiceAccount, through a projected token. Access policies +// name it with a "user:" prefix (see the access-policy subcommand). +func apiPrincipal(namespace string) string { + return "system:serviceaccount:" + namespace + ":" + apiName +} + +// buildAPIServiceAccount returns the ServiceAccount ate-env-api runs as. A +// dedicated account gives the control plane a stable principal to authorize; +// the namespace's default account would be shared with anything else there. +// ate-env-api never calls the Kubernetes API, thus no API token is mounted. +func buildAPIServiceAccount(cfg manifestConfig) *corev1.ServiceAccount { + automount := false + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ServiceAccount"}, + ObjectMeta: metav1.ObjectMeta{ + Name: apiName, + Namespace: cfg.namespace, + Labels: map[string]string{"app": apiName}, + }, + AutomountServiceAccountToken: &automount, + } +} + +// buildAccessPolicy returns the Substrate access policy that lets ate-env-api +// manage environments in one atespace: the editor role creates, suspends and +// deletes actors, and creates the per-image templates it derives. +func buildAccessPolicy(namespace string) *ateapipb.AccessPolicy { + return &ateapipb.AccessPolicy{ + Metadata: &ateapipb.ResourceMetadata{Name: "default"}, + Bindings: []*ateapipb.Binding{{ + Role: "editor", + Members: []string{"user:" + apiPrincipal(namespace)}, + }}, + } +} + // writeActorTemplate writes a single Substrate ActorTemplate as protojson-shaped YAML. func writeActorTemplate(w io.Writer, tmpl *ateapipb.ActorTemplate) error { + return writeProtoYAML(w, tmpl, "actor template") +} + +// writeProtoYAML writes a Substrate control-plane message as protojson-shaped +// YAML, the form kubectl-ate reads, without its server-managed status. +func writeProtoYAML(w io.Writer, msg proto.Message, what string) error { opts := protojson.MarshalOptions{ UseProtoNames: false, } - jsonBytes, err := opts.Marshal(tmpl) + jsonBytes, err := opts.Marshal(msg) if err != nil { - return fmt.Errorf("encoding actor template json: %w", err) + return fmt.Errorf("encoding %s json: %w", what, err) } var m map[string]any if err := json.Unmarshal(jsonBytes, &m); err != nil { - return fmt.Errorf("decoding actor template json: %w", err) + return fmt.Errorf("decoding %s json: %w", what, err) } delete(m, "status") prune(m) data, err := yaml.Marshal(m) if err != nil { - return fmt.Errorf("encoding actor template yaml: %w", err) + return fmt.Errorf("encoding %s yaml: %w", what, err) } _, err = w.Write(data) return err @@ -444,13 +517,13 @@ func buildWorkerPool(cfg manifestConfig) *atev1alpha1.WorkerPool { Spec: atev1alpha1.WorkerPoolSpec{ Replicas: cfg.replicas, WorkerImage: cfg.workerImage, - // The CRD requires exactly one sandbox class per pool, and - // actors only schedule onto workers of their template's class, - // so keep this in step with buildActorTemplate. The configRef is - // the SandboxConfig the workers run; the template names the same - // one in sandboxConfig.configName. + // The CRD requires exactly one sandbox class per pool. Since + // substrate#2780 the pool alone decides the sandbox: the + // ActorTemplate names none, and an actor restores onto a worker of + // its snapshot's class. The configRef is the SandboxConfig the + // workers run. SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{ - Name: atev1alpha1.SandboxClassGvisor, // the template's sandboxConfig.sandboxClass is the same string + Name: atev1alpha1.SandboxClassGvisor, ConfigRef: &atev1alpha1.SandboxConfigReference{Name: sandboxConfigName}, }}, }, @@ -489,10 +562,6 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate { // schema marks the field required, so say it explicitly. PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, - SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: string(atev1alpha1.SandboxClassGvisor), - ConfigName: sandboxConfigName, - }, Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{ {Name: "cpu", Quantity: orDefault(cfg.cpu, defaultActorCPU)}, {Name: "memory", Quantity: orDefault(cfg.memory, defaultActorMemory)}, @@ -548,6 +617,7 @@ func buildTemplate(cfg templateConfig) (*ateapipb.ActorTemplate, error) { // buildAPIDeployment returns the ate-env-api Deployment, pointed at the // in-cluster Substrate endpoints. func buildAPIDeployment(cfg manifestConfig) *appsv1.Deployment { + automountAPIToken := false labels := map[string]string{"app": apiName} replicas := cfg.apiReplicas tokenExpiration := int64(7200) @@ -564,6 +634,11 @@ func buildAPIDeployment(cfg manifestConfig) *appsv1.Deployment { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ + // The ServiceAccount is the principal the control plane + // authorizes (buildAccessPolicy); the projected ateapi token + // below is minted for it. + ServiceAccountName: apiName, + AutomountServiceAccountToken: &automountAPIToken, Containers: []corev1.Container{{ Name: apiName, Image: cfg.apiImage, diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go index 95f26594..8a0140e4 100644 --- a/cmd/ate-env/manifest_test.go +++ b/cmd/ate-env/manifest_test.go @@ -24,6 +24,7 @@ import ( "go.ate.dev/substrate/pkg/proto/ateapipb" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" ) func testManifestConfig() manifestConfig { @@ -84,8 +85,8 @@ func TestTemplateConfigResolveImages(t *testing.T) { func TestBuildManifests(t *testing.T) { cfg := testManifestConfig() objs := buildManifests(cfg) - if len(objs) != 4 { - t.Fatalf("got %d manifests, want 4", len(objs)) + if len(objs) != 5 { + t.Fatalf("got %d manifests, want 5", len(objs)) } ns := objs[0].(*corev1.Namespace) @@ -116,15 +117,15 @@ func TestBuildManifests(t *testing.T) { if sc.Name != atev1alpha1.SandboxClassGvisor || sc.ConfigRef == nil || sc.ConfigRef.Name != sandboxConfigName { t.Errorf("workerpool sandboxClasses[0] = %+v, want gvisor with configRef %q", sc, sandboxConfigName) } - // The pool's class and the template's are the same string since - // substrate#2586 made the proto field a string. A pool of the wrong class - // never hosts the template's actors. - tmplClass := buildActorTemplate(testTemplateConfig()).GetSandboxConfig().GetSandboxClass() - if tmplClass != string(sc.Name) { - t.Errorf("workerpool sandbox class %q != template sandbox class %q", sc.Name, tmplClass) + sa := objs[2].(*corev1.ServiceAccount) + if sa.Name != apiName || sa.Namespace != cfg.namespace || sa.AutomountServiceAccountToken == nil || *sa.AutomountServiceAccountToken { + t.Errorf("service account = %s/%s automount %v, want %s/%s with no API token", sa.Namespace, sa.Name, sa.AutomountServiceAccountToken, cfg.namespace, apiName) } - deployment := objs[2].(*appsv1.Deployment) + deployment := objs[3].(*appsv1.Deployment) + if pod := deployment.Spec.Template.Spec; pod.ServiceAccountName != apiName { + t.Errorf("api pod service account = %q, want %q (the principal the access policy names)", pod.ServiceAccountName, apiName) + } containers := deployment.Spec.Template.Spec.Containers if len(containers) != 1 || containers[0].Image != cfg.apiImage { t.Errorf("api containers = %+v, want one container with image %q", containers, cfg.apiImage) @@ -133,7 +134,7 @@ func TestBuildManifests(t *testing.T) { t.Errorf("api replicas = %d, want 1", *deployment.Spec.Replicas) } - service := objs[3].(*corev1.Service) + service := objs[4].(*corev1.Service) if service.Spec.Selector["app"] != apiName { t.Errorf("api service selector = %v, want app=%s", service.Spec.Selector, apiName) } @@ -165,9 +166,6 @@ func TestBuildActorTemplate(t *testing.T) { if got := template.GetSnapshotConfig().GetGoldenSnapshotConfig(); got != nil { t.Errorf("golden snapshot config = %v, want nil (omitted means enabled)", got) } - if got := template.GetSandboxConfig().GetConfigName(); got != sandboxConfigName { - t.Errorf("sandbox config name = %q, want %q (must match the WorkerPool configRef)", got, sandboxConfigName) - } if template.GetWorkerSelector() != nil { t.Errorf("worker selector = %v, want nil", template.WorkerSelector) } @@ -179,9 +177,6 @@ func TestBuildActorTemplate(t *testing.T) { if got := limits(template); got["cpu"] != defaultActorCPU || got["memory"] != defaultActorMemory || len(got) != 2 { t.Errorf("resources.limits = %v, want cpu=%s memory=%s", got, defaultActorCPU, defaultActorMemory) } - if got := template.GetSandboxConfig().GetSandboxClass(); got != "gvisor" { - t.Errorf("sandbox class = %q, want gvisor (Substrate accepts gvisor or microvm)", got) - } } func limits(tmpl *ateapipb.ActorTemplate) map[string]string { @@ -250,7 +245,7 @@ func TestBuildManifestsCustomAPIPort(t *testing.T) { cfg.apiPort = 9999 objs := buildManifests(cfg) - deployment := objs[2].(*appsv1.Deployment) + deployment := objs[3].(*appsv1.Deployment) container := deployment.Spec.Template.Spec.Containers[0] if got := container.Args[1]; got != "0.0.0.0:9999" { t.Errorf("api -listen arg = %q, want 0.0.0.0:9999", got) @@ -262,7 +257,7 @@ func TestBuildManifestsCustomAPIPort(t *testing.T) { t.Errorf("api probe port = %d, want 9999", got) } - service := objs[3].(*corev1.Service) + service := objs[4].(*corev1.Service) if service.Spec.Ports[0].Port != 9999 || service.Spec.Ports[0].TargetPort.IntValue() != 9999 { t.Errorf("api service ports = %+v, want 9999 -> 9999", service.Spec.Ports) } @@ -275,13 +270,15 @@ func TestWriteManifests(t *testing.T) { } out := buf.String() - if got := strings.Count(out, "\n---\n"); got != 3 { - t.Errorf("got %d document separators, want 3:\n%s", got, out) + if got := strings.Count(out, "\n---\n"); got != 4 { + t.Errorf("got %d document separators, want 4:\n%s", got, out) } for _, want := range []string{ "kind: Namespace", "ate.dev/api-access: \"true\"", "kind: WorkerPool", + "kind: ServiceAccount", + "serviceAccountName: ate-env-api", "kind: Deployment", "kind: Service", "sandboxClasses:", @@ -368,11 +365,15 @@ func TestManifestTemplateCommandExecution(t *testing.T) { if !strings.Contains(out, "image: example.com/guest@sha256:aaaa") || !strings.Contains(out, "storageLocation: gs://bucket/prefix/") { t.Errorf("expected guest image and storage location in output:\n%s", out) } - for _, want := range []string{"sandboxClass: gvisor", "name: cpu\n quantity: \"4\"", "name: memory\n quantity: 8Gi"} { + for _, want := range []string{"name: cpu\n quantity: \"4\"", "name: memory\n quantity: 8Gi"} { if !strings.Contains(out, want) { t.Errorf("output lacks %q:\n%s", want, out) } } + // Substrate rejects a template with sandboxConfig since substrate#2780. + if strings.Contains(out, "sandboxConfig") || strings.Contains(out, "sandboxClass") { + t.Errorf("template names a sandbox; the worker pool decides it now:\n%s", out) + } } const testHex64 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" @@ -531,3 +532,34 @@ func TestImageFlagsDefaultToReleaseImages(t *testing.T) { t.Errorf("missing --worker-image should point at the Substrate repo, got %v", err) } } + +func TestManifestAccessPolicyCommand(t *testing.T) { + for _, ns := range []string{"", "team-env"} { + cmd := newManifestCommand() + var out bytes.Buffer + cmd.SetOut(&out) + args := []string{"access-policy"} + if ns != "" { + args = append(args, "--namespace", ns) + } + cmd.SetArgs(args) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } + want := "user:system:serviceaccount:" + map[bool]string{true: "ate-env", false: ns}[ns == ""] + ":ate-env-api" + var m map[string]any + if err := yaml.Unmarshal(out.Bytes(), &m); err != nil { + t.Fatalf("%q: %v", out.String(), err) + } + md, _ := m["metadata"].(map[string]any) + bindings, _ := m["bindings"].([]any) + if md["name"] != "default" || md["atespace"] != nil || len(bindings) != 1 { + t.Fatalf("policy = %v, want metadata.name default, no atespace, one binding", m) + } + b := bindings[0].(map[string]any) + members, _ := b["members"].([]any) + if b["role"] != "editor" || len(members) != 1 || members[0] != want { + t.Errorf("binding = %v, want editor for %s", b, want) + } + } +} diff --git a/go.mod b/go.mod index f611624d..a911b31c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/google/uuid v1.6.0 github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/spf13/cobra v1.10.2 - go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748 + go.ate.dev/substrate v0.5.1-0.20261018212731-ddb46651d605 google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 k8s.io/api v0.37.1 diff --git a/go.sum b/go.sum index edf211fe..166d15cd 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= -go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748 h1:QrTauJhnfn0FL0t/GLzVo4bmeVUYMT6e4x8QSaQF8s0= -go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748/go.mod h1:EktaeScxg724tF4cjtLV8kdz2xN3FH8NOTk/A95jSBY= +go.ate.dev/substrate v0.5.1-0.20261018212731-ddb46651d605 h1:0hbCfZd53YPGYhtBM0mHs/UCz66J47l0GTeMscaIezk= +go.ate.dev/substrate v0.5.1-0.20261018212731-ddb46651d605/go.mod h1:EktaeScxg724tF4cjtLV8kdz2xN3FH8NOTk/A95jSBY= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go index d7050e23..fc5d831b 100644 --- a/internal/apiservice/imagetemplate_test.go +++ b/internal/apiservice/imagetemplate_test.go @@ -39,7 +39,6 @@ func modeABase() *ateapipb.ActorTemplate { WakeupProbe: &ateapipb.WakeupProbe{HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}}, }}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://b/p/", PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{ {Name: "cpu", Quantity: "2"}, {Name: "memory", Quantity: "4Gi"}, }}, @@ -124,10 +123,9 @@ func TestDeriveImageTemplateFromGuestImageBase(t *testing.T) { // Everything else carries over. if !proto.Equal(got.GetWorkerSelector(), base.GetWorkerSelector()) || !proto.Equal(got.GetSnapshotConfig(), base.GetSnapshotConfig()) || - !proto.Equal(got.GetSandboxConfig(), base.GetSandboxConfig()) || !proto.Equal(c.GetWakeupProbe(), base.GetContainers()[0].GetWakeupProbe()) || len(c.GetEnv()) != 1 { - t.Error("worker selector, snapshots, sandbox config, readyz or env did not carry over") + t.Error("worker selector, snapshots, readyz or env did not carry over") } } diff --git a/internal/apiservice/server_test.go b/internal/apiservice/server_test.go index 9ecccd5c..a9dfae2d 100644 --- a/internal/apiservice/server_test.go +++ b/internal/apiservice/server_test.go @@ -599,7 +599,6 @@ func seedGuestTemplate(control *fakecontrol.Server, atespace, name string) { WakeupProbe: &ateapipb.WakeupProbe{HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}}, }}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://bucket/ate-env/"}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, }) } diff --git a/internal/ate/client.go b/internal/ate/client.go index 344e953e..b21d88ba 100644 --- a/internal/ate/client.go +++ b/internal/ate/client.go @@ -296,8 +296,18 @@ func (c *Client) Get(ctx context.Context, atespace, id string) (*ateapipb.Actor, // Suspend checkpoints and stops the actor with ID in atespace. // If atespace is empty, DefaultAtespace is used. +// +// The suspend is synchronous: it returns once the snapshot is in durable +// storage, thus a suspended environment survives the loss of its node. +// Substrate's default became asynchronous in substrate#2805 (the snapshot +// stays on the node and uploads in the background); ate-env keeps the +// durable behavior it always had. func (c *Client) Suspend(ctx context.Context, atespace, id string) error { - if _, err := c.control.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: c.ref(atespace, id)}); err != nil { + req := &ateapipb.SuspendActorRequest{ + Actor: c.ref(atespace, id), + Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}, + } + if _, err := c.control.SuspendActor(ctx, req); err != nil { return fmt.Errorf("ate: suspending %q: %w", id, wrapGRPCError(err)) } return nil diff --git a/internal/ate/client_test.go b/internal/ate/client_test.go index 727e5fcf..e2786aa1 100644 --- a/internal/ate/client_test.go +++ b/internal/ate/client_test.go @@ -118,6 +118,11 @@ func TestSuspend(t *testing.T) { if st := f.control.Status("sb-susp"); st != ateapipb.ActorState_ACTOR_STATE_SUSPENDED { t.Fatalf("status = %v, want SUSPENDED", st) } + // Substrate's default is asynchronous since substrate#2805; ate-env asks + // for a durable suspend. + if got := f.control.SuspendModes; len(got) != 1 || got[0] != ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS { + t.Fatalf("suspend modes = %v, want one SYNCHRONOUS", got) + } } func TestSuspendMissing(t *testing.T) { diff --git a/internal/internaltest/fakecontrol/fakecontrol.go b/internal/internaltest/fakecontrol/fakecontrol.go index 1a87acc1..0adf3fba 100644 --- a/internal/internaltest/fakecontrol/fakecontrol.go +++ b/internal/internaltest/fakecontrol/fakecontrol.go @@ -51,6 +51,8 @@ type Server struct { // Substrate does for an identity allowed to use but not create atespaces. DenyCreateAtespace bool templates map[string]*ateapipb.ActorTemplate + // SuspendModes records the durability mode of each SuspendActor call. + SuspendModes []ateapipb.DurabilityMode } // New returns an empty fake control server. @@ -272,6 +274,7 @@ func (s *Server) SuspendActor(ctx context.Context, req *ateapipb.SuspendActorReq if err != nil { return nil, err } + s.SuspendModes = append(s.SuspendModes, req.GetOptions().GetMode()) s.suspend(a) return &ateapipb.SuspendActorResponse{Actor: clone(a)}, nil } @@ -292,18 +295,6 @@ func (s *Server) suspend(a *ateapipb.Actor) { } } -func (s *Server) PauseActor(ctx context.Context, req *ateapipb.PauseActorRequest) (*ateapipb.PauseActorResponse, error) { - s.mu.Lock() - defer s.mu.Unlock() - a, err := s.get(req.GetActor()) - if err != nil { - return nil, err - } - // Substrate reports a paused actor as SUSPENDED (substrate#2620). - a.Status = &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED} - return &ateapipb.PauseActorResponse{Actor: clone(a)}, nil -} - func (s *Server) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) { s.mu.Lock() defer s.mu.Unlock() From 40a6c63d5763ba6fc2825b996a9a8c3c73b200fb Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:30:18 -0700 Subject: [PATCH 2/3] docs: Substrate ddb46651 boundary, durable suspend, the access-policy install step The README's compatibility note names ddb46651 and the 0443aec3 boundary (templates without sandboxConfig), says that ate-env-api suspends synchronously, and that Substrate enforces authorization. Step 1 now creates the atespace and the access policy before the manifest, with what to do when the atespace already has a policy. The upgrade notes cover the access policy (without it ate-env-api fails at startup), old base templates (Substrate drops their sandboxConfig when it stores them) and the earlier limits boundary. The rootfs-image docs no longer say that a template carries a sandbox class. --- README.md | 61 +++++++++++++++++++++++++----------- docs/rootfs-images/DESIGN.md | 5 +-- docs/rootfs-images/README.md | 6 ++-- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index cf18493e..f660dd68 100644 --- a/README.md +++ b/README.md @@ -65,28 +65,51 @@ verification, pinning and how releases are cut. Prerequisites: a cluster with [Agent Substrate](https://github.com/agent-substrate/substrate) installed and a snapshots bucket. ate-env is built against Substrate -[`3a2ca623`](https://github.com/agent-substrate/substrate/commit/3a2ca623) (2026-10-16, Go module -`go.ate.dev/substrate`). The cluster needs a Substrate recent enough to accept what ate-env writes: -the WorkerPool declares `spec.sandboxClasses`, the ActorTemplate uses `snapshotConfig` and -`wakeupProbe`, names its sandbox class as the string `gvisor`, and declares CPU and memory limits. -That means Substrate at or after -[`63a17a92`](https://github.com/agent-substrate/substrate/commit/63a17a92) (2026-10-15), which also -renamed the worker image: `--worker-image` must be the `ateworker-gvisor` image published by that -Substrate install. An older Substrate rejects the string sandbox class (from -[`9ec49759`](https://github.com/agent-substrate/substrate/commit/9ec49759)) or the limits (from -[`c58f374a`](https://github.com/agent-substrate/substrate/commit/c58f374a)); use an ate-env from before -this bump with it. - -When you upgrade an existing cluster across that boundary, register your base templates again. -Substrate cannot update an ActorTemplate in place, and a base registered earlier has no CPU and memory -limits, thus each template `ate-env-api` derives from it for a rootfs image fails with -`resources: Required`. Delete the base and its derived templates (named `-<12 hex digits>`) with -`kubectl-ate delete actor-template`, then register the base again with `ate-env manifest template`. -Each environment created after that derives its template again, with the limits. +[`ddb46651`](https://github.com/agent-substrate/substrate/commit/ddb46651) (2026-10-18, Go module +`go.ate.dev/substrate`). The cluster needs Substrate at or after +[`0443aec3`](https://github.com/agent-substrate/substrate/commit/0443aec3) (2026-10-17): from that +commit the WorkerPool alone chooses the sandbox (`spec.sandboxClasses`), the ActorTemplate names +none, and Substrate rejects a template that still sends `sandboxConfig`, while an older Substrate +requires one. Use an ate-env from before this bump with an older Substrate. The templates also use +`snapshotConfig` and `wakeupProbe` and declare CPU and memory limits, and `--worker-image` must be the +`ateworker-gvisor` image published by the same Substrate install. + +`ate-env-api` suspends synchronously: a suspend returns once the snapshot is in object storage, thus a +suspended environment survives the loss of its node. Substrate's own default became asynchronous in +[`b40d34f1`](https://github.com/agent-substrate/substrate/commit/b40d34f1). + +Substrate enforces authorization by default. `ate-env-api` runs as its own ServiceAccount +(`ate-env/ate-env-api`) and needs the editor role on each atespace it creates environments in; the +atespace must exist first, because creating one takes a global owner. Step 1 sets both up. + +When you upgrade an existing cluster across these changes: + +- Create the atespace and the access policy (step 1) before you apply the new manifest. Without them + `ate-env-api` fails at startup with `PermissionDenied` while it checks the default atespace. +- Base templates registered before need no change: Substrate drops their old `sandboxConfig` when + it stores them. Register new bases with this `ate-env`. +- A base registered before the CPU and memory limits (Substrate before + [`c58f374a`](https://github.com/agent-substrate/substrate/commit/c58f374a)) still needs registering + again: Substrate cannot update an ActorTemplate in place, thus each template `ate-env-api` derives + from it fails with `resources: Required`. Delete the base and its derived templates (named + `-<12 hex digits>`) with `kubectl-ate delete actor-template`, then register the base again. ### 1. Deploy the Kubernetes Resources -Deploy the namespace, worker pool, and API service: +Give `ate-env-api` access to its atespace, once, as an owner of the atespace (on a stock install the +default `kubectl-ate` identity is a bootstrap owner): + +```bash +kubectl ate create atespace ate-env +ate-env manifest access-policy | kubectl ate create access-policy -a ate-env -f - +``` + +An atespace has one access policy. If `ate-env` already has one, add the binding that +`ate-env manifest access-policy` prints to it with `kubectl ate get access-policy` and +`kubectl ate update access-policy`. Repeat both commands for each other atespace you create +environments in. + +Deploy the namespace, worker pool, ServiceAccount and API service: ```bash # With a release binary the API image defaults to the release's. The worker diff --git a/docs/rootfs-images/DESIGN.md b/docs/rootfs-images/DESIGN.md index fd9dfcf6..fe976752 100644 --- a/docs/rootfs-images/DESIGN.md +++ b/docs/rootfs-images/DESIGN.md @@ -146,8 +146,9 @@ piece, and it belongs with the OpenSandbox backend that needs it. ## Security -- The rootfs image is run with the same sandbox class and configuration as the - base, so injection changes what runs, not how it is isolated. +- The rootfs image runs in the same sandbox as any other actor of the worker + pool (the pool's sandbox class decides it; the template names none), so + injection changes what runs, not how it is isolated. - The guest volume is read-only. The rootfs image cannot alter the guest. - Digest pinning is enforced for both images at every entry point. A tag could be re-pointed between the golden bake and a later cold boot, which would diff --git a/docs/rootfs-images/README.md b/docs/rootfs-images/README.md index 2843f20c..a0cf9265 100644 --- a/docs/rootfs-images/README.md +++ b/docs/rootfs-images/README.md @@ -19,7 +19,7 @@ process = /ate/ko-app/ate-env-guest, from a second image mounted read-only at / ``` Substrate materializes both at actor start. Everything else about the template -(worker selector, snapshot settings, sandbox class, resources, readiness probe) +(worker selector, snapshot settings, resources, readiness probe) is the same as for a guest-image template, and the guest behaves identically: the environment's `shell`, process and file APIs, MCP, and suspend and resume all work unchanged. @@ -63,7 +63,7 @@ ate-env create job1 --template py312 `--rootfs-image` makes the rootfs image the container and the guest an image volume; `--workspace` is passed to the guest as `-workspace` in either mode. The printed template is plain YAML you can edit before registering it (resources, -worker selector, sandbox class). +worker selector). The sandbox class is the worker pool's, not the template's. ## Option B: create from an image on demand @@ -137,7 +137,7 @@ runnable example in [`examples/runtime-layers`](../../examples/runtime-layers). - Bumping the guest means a new base template (templates are immutable), and a new base name yields new derived names. Existing derived templates keep the guest they were built with. -- Resources, sandbox class and worker selector come from the base; a rootfs image +- Resources and worker selector come from the base, and the sandbox from the worker pool; a rootfs image that needs more is served by registering its own template (Option A) or a second base. - The rootfs image's existence is not checked at create time. A bad reference is From 2c780cd94891c134cd10375a30a5e1e662ee2c2d Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Sun, 18 Oct 2026 19:34:54 -0700 Subject: [PATCH 3/3] ate-env manifest: templates select their WorkerPool by label Since substrate#2780 the template names no sandbox class, and Substrate's release note flags that a template with no workerSelector whose actor has no snapshot can be placed on a pool of any class. ate-env templates had no selector. On a cluster with more than one pool (another team's, or an OpenShell microvm pool), a cold-booting environment could land on the wrong one. The WorkerPool now carries `ate-env/workerpool: `; Substrate copies a pool's labels to its workers. Each template selects that label, through a new --workerpool flag on `ate-env manifest template` that defaults to