objectstore-plugin: storage settings come from ate-storage-config

cmd/ate-setup/internal/steps/deploy.go

@@ -120,12 +120,15 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error {
 		return err
 	}
 
-	// Ahead of the bundle below, for the same reason as the namespace: every
-	// workload pulls this ConfigMap in via envFrom, and a container whose
+	// Ahead of the bundle below, for the same reason as the namespace: the
+	// workloads pull these ConfigMaps in via envFrom, and a container whose
 	// envFrom target is missing will not start.
 	if err := e.applyOtelConfig(ctx); err != nil {
 		return err
 	}
+	if err := e.applyStorageConfig(ctx); err != nil {
+		return err
+	}
 
 	// Resolved before the bundle apply: adopting a Cloud SQL instance reads
 	// the ConfigMap that EnsureAPIServerPrerequisites has already rewritten,
@@ -319,6 +322,9 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error {
 	if err := e.applyOtelEndpointOverride(ctx); err != nil {
 		return err
 	}
+	if err := e.applyStorageConfig(ctx); err != nil {
+		return err
+	}
 	postgres, err := e.planPostgres(ctx)
 	if err != nil {
 		return err
@@ -504,6 +510,9 @@ func (e *Env) DeployAtelet(ctx context.Context) error {
 	if err := e.applyOtelEndpointOverride(ctx); err != nil {
 		return err
 	}
+	if err := e.applyStorageConfig(ctx); err != nil {
+		return err
+	}
 
 	manifest, err := e.render(e.ateletManifestPath())
 	if err != nil {

cmd/ate-setup/internal/steps/deploy_test.go

@@ -185,9 +185,10 @@ func TestMissingCRDsLeavesPresentOnesAlone(t *testing.T) {
 	}
 }
 
-// A single-component redeploy has to apply what deploy ate-system does. On
-// kind the bundle points the object-store plugins at rustfs, and a redeploy
-// from a base file would fall back to GCS and never start.
+// A single-component redeploy has to apply what deploy ate-system does: the
+// same workloads, with the kind patches on kind, and the same storage settings
+// from storageConfigPath. Applying the base storage settings on kind would
+// point the object-store plugins at GCS.
 func TestComponentManifestsMatchSystemBundle(t *testing.T) {
 	for _, tc := range []struct {
 		router string
@@ -227,5 +228,20 @@ func TestComponentManifestsMatchSystemBundle(t *testing.T) {
 				t.Errorf("router=%s kind=%v: %s renders a different %s/%s than the system bundle (-bundle +standalone):\n%s", tc.router, tc.kind, component.path, component.kind, component.name, diff)
 			}
 		}
+		storage, err := kube.LoadPath(e.storageConfigPath())
+		if err != nil {
+			t.Fatalf("LoadPath(%s): %v", e.storageConfigPath(), err)
+		}
+		if len(storage) == 0 {
+			t.Fatalf("%s holds no objects", e.storageConfigPath())
+		}
+		for _, got := range storage {
+			want := findObject(bundle, got.GetKind(), got.GetName())
+			if want == nil {
+				t.Errorf("router=%s kind=%v: the system bundle has no %s", tc.router, tc.kind, kube.Describe(got))
+			} else if diff := cmp.Diff(want.Object, got.Object); diff != "" {
+				t.Errorf("router=%s kind=%v: %s holds a different %s than the system bundle (-bundle +standalone):\n%s", tc.router, tc.kind, e.storageConfigPath(), kube.Describe(got), diff)
+			}
+		}
 	}
 }

cmd/ate-setup/internal/steps/overlay.go

@@ -545,6 +545,22 @@ func (e *Env) applyOtelConfig(ctx context.Context) error {
 	return e.Kube.ApplyPath(ctx, e.otelConfigPath())
 }
 
+// storageConfigPath returns the environment's ate-storage-config ConfigMap,
+// with the ate-storage-credentials Secret on kind. The objectstore-plugin
+// sidecars of ate-api-server and atelet read both via envFrom. As with
+// otelConfigPath, the base file on a kind cluster would point them at GCS.
+func (e *Env) storageConfigPath() string {
+	if e.Cfg.Kind {
+		return e.Cfg.Manifest("kind", "ate-storage-config.yaml")
+	}
+	return e.Cfg.Manifest("ate-storage-config.yaml")
+}
+
+// applyStorageConfig applies the environment's storage settings.
+func (e *Env) applyStorageConfig(ctx context.Context) error {
+	return e.Kube.ApplyPath(ctx, e.storageConfigPath())
+}
+
 // otelConfigMap is the ConfigMap every control plane component reads its
 // telemetry settings from through envFrom.
 const otelConfigMap = "ate-otel-config"

cmd/ate-setup/internal/steps/overlay_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"maps"
 	"os"
 	"reflect"
 	"slices"
@@ -30,6 +31,7 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	dynamicfake "k8s.io/client-go/dynamic/fake"
@@ -594,9 +596,8 @@ func dataPlaneWorkloads(t *testing.T, manifest []byte) (daemonSets map[string]ap
 
 // Under --fake-data-plane every path that applies atelet or ate-controller
 // carries the fake data plane: fake-atelet in atelet's DaemonSet, with the
-// storage settings the overlay gives atelet's objectstore-plugin, whose
-// writes it takes over, ate-controller at zero replicas, and fake-workersync
-// beside them.
+// storage settings of atelet's objectstore-plugin, whose writes it takes
+// over, ate-controller at zero replicas, and fake-workersync beside them.
 func TestRenderFakeDataPlane(t *testing.T) {
 	root := repoRoot(t)
 	const ateletDS = "atelet"
@@ -606,8 +607,8 @@ func TestRenderFakeDataPlane(t *testing.T) {
 		path           func(e *Env) string
 		wantAtelet     bool
 		wantController bool
-		// wantStorage is the ATE_STORAGE_BACKEND the overlay gives
-		// atelet's objectstore-plugin.
+		// wantStorage is the ATE_STORAGE_BACKEND atelet's
+		// objectstore-plugin reads.
 		wantStorage string
 	}{
 		{
@@ -702,7 +703,7 @@ func TestRenderFakeDataPlane(t *testing.T) {
 					t.Errorf("atelet container image = %q, want %q", c.Image, want)
 				}
 				// fake-atelet runs with its defaults, tuned only through the
-				// ConfigMap a benchmark writes, beside atelet's own envFrom.
+				// ConfigMap a benchmark writes, beside the sidecar's envFrom.
 				if len(c.Args) != 0 {
 					t.Errorf("fake-atelet args = %v, want none", c.Args)
 				}
@@ -718,7 +719,7 @@ func TestRenderFakeDataPlane(t *testing.T) {
 				if !slices.ContainsFunc(c.EnvFrom, func(src corev1.EnvFromSource) bool {
 					return src.ConfigMapRef != nil && src.ConfigMapRef.Name == otelConfigMap
 				}) {
-					t.Errorf("fake-atelet envFrom = %+v, lost atelet's %s", c.EnvFrom, otelConfigMap)
+					t.Errorf("fake-atelet envFrom = %+v, lost %s", c.EnvFrom, otelConfigMap)
 				}
 				for _, p := range c.Ports {
 					if p.HostPort != 0 {
@@ -735,10 +736,7 @@ func TestRenderFakeDataPlane(t *testing.T) {
 						t.Errorf("fake-atelet probe %s targets port %q, which the container does not declare", probe.HTTPGet.Path, probe.HTTPGet.Port.StrVal)
 					}
 				}
-				env := map[string]string{}
-				for _, v := range c.Env {
-					env[v.Name] = v.Value
-				}
+				env := storageEnv(storageSettings(t, e), c)
 				if env["ATE_STORAGE_BACKEND"] != tc.wantStorage {
 					t.Errorf("fake-atelet ATE_STORAGE_BACKEND = %q, want objectstore-plugin's %q", env["ATE_STORAGE_BACKEND"], tc.wantStorage)
 				}
@@ -1511,3 +1509,147 @@ func TestRemoveFakeWorkersync(t *testing.T) {
 		})
 	}
 }
+
+// storageSettings returns the keys of each ConfigMap and Secret in e's
+// storage settings, by kind and name.
+func storageSettings(t *testing.T, e *Env) map[string]map[string]string {
+	t.Helper()
+	objs, err := kube.LoadPath(e.storageConfigPath())
+	if err != nil {
+		t.Fatalf("LoadPath(%s): %v", e.storageConfigPath(), err)
+	}
+	settings := map[string]map[string]string{}
+	for _, obj := range objs {
+		field := "data"
+		if obj.GetKind() == "Secret" {
+			field = "stringData"
+		}
+		data, _, err := unstructured.NestedStringMap(obj.Object, field)
+		if err != nil {
+			t.Fatalf("%s: %v", kube.Describe(obj), err)
+		}
+		settings[obj.GetKind()+"/"+obj.GetName()] = data
+	}
+	return settings
+}
+
+// storageEnv returns the storage settings c runs with: its envFrom sources
+// resolved from settings, then its env, which overrides them.
+func storageEnv(settings map[string]map[string]string, c corev1.Container) map[string]string {
+	env := map[string]string{}
+	for _, src := range c.EnvFrom {
+		switch {
+		case src.ConfigMapRef != nil:
+			maps.Copy(env, settings["ConfigMap/"+src.ConfigMapRef.Name])
+		case src.SecretRef != nil:
+			maps.Copy(env, settings["Secret/"+src.SecretRef.Name])
+		}
+	}
+	for _, v := range c.Env {
+		env[v.Name] = v.Value
+	}
+	maps.DeleteFunc(env, func(k, _ string) bool { return !isStorageEnv(k) })
+	return env
+}
+
+func isStorageEnv(name string) bool {
+	return name == "ATE_STORAGE_BACKEND" || strings.HasPrefix(name, "AWS_")
+}
+
+// The objectstore-plugin sidecars read their storage settings from
+// ate-storage-config, which an inline copy would override. ATE_STORAGE_BACKEND
+// must not reach the main containers: ate-api-server and atelet exit if it is
+// set.
+func TestObjectstorePluginsReadStorageConfig(t *testing.T) {
+	for _, tc := range []struct {
+		kind        bool
+		wantBackend string
+	}{
+		{false, "gcs"},
+		{true, "s3"},
+	} {
+		e := &Env{Cfg: &config.Config{Root: repoRoot(t), Kind: tc.kind}}
+		manifest, err := e.render(e.Cfg.Path(SystemOverlay(e.Cfg)))
+		if err != nil {
+			t.Fatalf("kind=%v: render: %v", tc.kind, err)
+		}
+		settings := storageSettings(t, e)
+		daemonSets, deployments := dataPlaneWorkloads(t, manifest)
+		for name, pod := range map[string]corev1.PodSpec{
+			"ate-api-server": deployments["ate-api-server"].Spec.Template.Spec,
+			"atelet":         daemonSets["atelet"].Spec.Template.Spec,
+		} {
+			if len(pod.InitContainers) == 0 || pod.InitContainers[0].Name != "objectstore-plugin" {
+				t.Fatalf("kind=%v: %s has no objectstore-plugin sidecar", tc.kind, name)
+			}
+			sidecar := pod.InitContainers[0]
+			for _, v := range sidecar.Env {
+				if isStorageEnv(v.Name) {
+					t.Errorf("kind=%v: %s objectstore-plugin sets %s inline", tc.kind, name, v.Name)
+				}
+			}
+			// Only kind ships the Secret.
+			if !slices.ContainsFunc(sidecar.EnvFrom, func(src corev1.EnvFromSource) bool {
+				ref := src.SecretRef
+				return ref != nil && ref.Name == "ate-storage-credentials" && ref.Optional != nil && *ref.Optional
+			}) {
+				t.Errorf("kind=%v: %s objectstore-plugin envFrom = %+v, want the optional ate-storage-credentials Secret", tc.kind, name, sidecar.EnvFrom)
+			}
+			if got := storageEnv(settings, sidecar)["ATE_STORAGE_BACKEND"]; got != tc.wantBackend {
+				t.Errorf("kind=%v: %s objectstore-plugin ATE_STORAGE_BACKEND = %q, want %q", tc.kind, name, got, tc.wantBackend)
+			}
+			for _, c := range pod.Containers {
+				if _, ok := storageEnv(settings, c)["ATE_STORAGE_BACKEND"]; ok {
+					t.Errorf("kind=%v: %s container %s gets ATE_STORAGE_BACKEND", tc.kind, name, c.Name)
+				}
+			}
+		}
+	}
+}
+
+// kind's rustfs and its bucket-init Job set the same keys inline, so they have
+// to match what the sidecars read.
+func TestRustfsMatchesKindStorageConfig(t *testing.T) {
+	e := &Env{Cfg: &config.Config{Root: repoRoot(t), Kind: true}}
+	settings := storageSettings(t, e)
+	want := map[string]string{}
+	maps.Copy(want, settings["ConfigMap/ate-storage-config"])
+	maps.Copy(want, settings["Secret/ate-storage-credentials"])
+
+	objs, err := kube.LoadPath(e.Cfg.Manifest("kind", "rustfs.yaml"))
+	if err != nil {
+		t.Fatalf("LoadPath: %v", err)
+	}
+	env := func(kind, name string) map[string]string {
+		obj := findObject(objs, kind, name)
+		if obj == nil {
+			t.Fatalf("rustfs.yaml has no %s/%s", kind, name)
+		}
+		var workload struct {
+			Spec struct {
+				Template corev1.PodTemplateSpec `json:"template"`
+			} `json:"spec"`
+		}
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &workload); err != nil {
+			t.Fatalf("%s: %v", kube.Describe(obj), err)
+		}
+		got := map[string]string{}
+		for _, c := range workload.Spec.Template.Spec.Containers {
+			for _, v := range c.Env {
+				got[v.Name] = v.Value
+			}
+		}
+		return got
+	}
+
+	job := env("Job", "rustfs-bucket-init")
+	for _, k := range []string{"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION", "AWS_ENDPOINT_URL"} {
+		if job[k] != want[k] {
+			t.Errorf("rustfs-bucket-init %s = %q, want %q", k, job[k], want[k])
+		}
+	}
+	server := env("Deployment", "rustfs")
+	if server["RUSTFS_ACCESS_KEY"] != want["AWS_ACCESS_KEY_ID"] || server["RUSTFS_SECRET_KEY"] != want["AWS_SECRET_ACCESS_KEY"] {
+		t.Errorf("rustfs keys = %q/%q, want the ate-storage-credentials keys", server["RUSTFS_ACCESS_KEY"], server["RUSTFS_SECRET_KEY"])
+	}
+}

docs/architecture.md

@@ -329,7 +329,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov
 
   * **Lifecycle Management**: The `ateworker` process invokes the sandbox runtime to checkpoint or restore processes within the physical pod boundaries — `runsc` for gVisor, or the Kata + Cloud Hypervisor stack for microVMs. (Note: the gVisor backend currently requires a `runsc` version with the `--allow-connected-on-save` flag to work around a bug in networking resumption during checkpointing.)
 
-  * **Storage Mover**: External snapshots move through an *object-store plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/objectstore-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. The same `atelet` sidecar serves `AssetProvider`, which fetches every sandbox asset; `atelet` reads a `gs://` asset anonymously only when the plugin cannot serve it, which covers public buckets such as `gs://gvisor`. The plugin stages each download where `atelet` cannot read it, and copies it into `atelet`'s cache only if its sha256 matches; it refuses any URL under a snapshot location. `atelet` verifies the sha256 again and has no object storage credentials or settings of its own.
+  * **Storage Mover**: External snapshots move through an *object-store plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/objectstore-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which. Both sidecars read these settings from the `ate-storage-config` ConfigMap, and static S3 keys from the optional `ate-storage-credentials` Secret; see `manifests/ate-install/ate-storage-config.yaml`, and `manifests/ate-install/kind/ate-storage-config.yaml` for an S3 example. The same `atelet` sidecar serves `AssetProvider`, which fetches every sandbox asset; `atelet` reads a `gs://` asset anonymously only when the plugin cannot serve it, which covers public buckets such as `gs://gvisor`. The plugin stages each download where `atelet` cannot read it, and copies it into `atelet`'s cache only if its sha256 matches; it refuses any URL under a snapshot location. `atelet` verifies the sha256 again and has no object storage credentials or settings of its own.
 
   * **Sandbox Asset Cache**: `atelet` keeps one node-local, content-addressed cache of the sandbox binaries every actor runs on and gives each actor hard links into it, prewarms each local worker's default runtime and keeps it linked under the worker's directory, and evicts to a byte budget without taking anything an actor or a worker still uses. See the [Sandbox Asset Cache](sandbox-asset-cache.md) page.
 

internal/e2e/README.md

@@ -87,8 +87,8 @@ lane that forgets cannot silently skip every preview test; see
 ## GCS-only suites
 
 `internal/e2e/suites/snapshotcredentials` runs only against an install that
-stores snapshots in GCS, which it reads from the objectstore-plugin sidecar on
-ate-api-server, and skips on any other backend, such as the kind lane's
+stores snapshots in GCS, which it reads from the `ate-storage-config`
+ConfigMap, and skips on any other backend, such as the kind lane's
 S3-compatible service. A lane that is meant to be GCS sets
 `E2E_REQUIRE_GCS=true`, which turns that skip into a failure, so a lane
 misconfigured as S3 cannot pass by running nothing.

internal/e2e/backend.go

@@ -28,7 +28,7 @@ const GCSRequiredEnv = "E2E_REQUIRE_GCS"
 func GCSRequired() bool { return os.Getenv(GCSRequiredEnv) == "true" }
 
 // RequireGCSBackend skips t unless backend, the install's snapshot backend
-// as the objectstore-plugin reports it, is GCS. Under GCSRequired it fails
+// as its ate-storage-config gives it, is GCS. Under GCSRequired it fails
 // instead.
 func RequireGCSBackend(t testing.TB, backend string) {
 	t.Helper()

internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go

@@ -51,24 +51,17 @@ import (
 const probeTemplate = "probe"
 
 // requireGCS skips, or fails under e2e.GCSRequired, unless the install under
-// test stores snapshots in GCS. The backend is the objectstore-plugin
-// sidecar's ATE_STORAGE_BACKEND on ate-api-server.
+// test stores snapshots in GCS. The backend is ATE_STORAGE_BACKEND in the
+// ate-storage-config ConfigMap the objectstore-plugin sidecars read.
 func requireGCS(t *testing.T, ctx context.Context, clients *e2e.Clients) {
 	t.Helper()
-	deploy, err := clients.K8s.AppsV1().Deployments(e2e.SystemNamespace()).Get(ctx, e2e.ResourceName("ate-api-server"), metav1.GetOptions{})
+	cm, err := clients.K8s.CoreV1().ConfigMaps(e2e.SystemNamespace()).Get(ctx, e2e.ResourceName("ate-storage-config"), metav1.GetOptions{})
 	if err != nil {
-		t.Fatalf("getting the ate-api-server Deployment: %v", err)
+		t.Fatalf("getting the ate-storage-config ConfigMap: %v", err)
 	}
-	backend := "gcs"
-	for _, c := range deploy.Spec.Template.Spec.InitContainers {
-		if c.Name != "objectstore-plugin" {
-			continue
-		}
-		for _, env := range c.Env {
-			if env.Name == "ATE_STORAGE_BACKEND" && env.Value != "" {
-				backend = env.Value
-			}
-		}
+	backend := cm.Data["ATE_STORAGE_BACKEND"]
+	if backend == "" {
+		backend = "gcs"
 	}
 	e2e.RequireGCSBackend(t, backend)
 }