diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e235e96b0a..d25424636a 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -120,12 +120,15 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return err } - // Ahead of the bundle below, for the same reason as the namespace: every - // workload pulls this ConfigMap in via envFrom, and a container whose + // Ahead of the bundle below, for the same reason as the namespace: the + // workloads pull these ConfigMaps in via envFrom, and a container whose // envFrom target is missing will not start. if err := e.applyOtelConfig(ctx); err != nil { return err } + if err := e.applyStorageConfig(ctx); err != nil { + return err + } // Resolved before the bundle apply: adopting a Cloud SQL instance reads // the ConfigMap that EnsureAPIServerPrerequisites has already rewritten, @@ -319,6 +322,9 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error { if err := e.applyOtelEndpointOverride(ctx); err != nil { return err } + if err := e.applyStorageConfig(ctx); err != nil { + return err + } postgres, err := e.planPostgres(ctx) if err != nil { return err @@ -504,6 +510,9 @@ func (e *Env) DeployAtelet(ctx context.Context) error { if err := e.applyOtelEndpointOverride(ctx); err != nil { return err } + if err := e.applyStorageConfig(ctx); err != nil { + return err + } manifest, err := e.render(e.ateletManifestPath()) if err != nil { diff --git a/cmd/ate-setup/internal/steps/deploy_test.go b/cmd/ate-setup/internal/steps/deploy_test.go index 12c504fdb8..1025b56959 100644 --- a/cmd/ate-setup/internal/steps/deploy_test.go +++ b/cmd/ate-setup/internal/steps/deploy_test.go @@ -185,9 +185,10 @@ func TestMissingCRDsLeavesPresentOnesAlone(t *testing.T) { } } -// A single-component redeploy has to apply what deploy ate-system does. On -// kind the bundle points the object-store plugins at rustfs, and a redeploy -// from a base file would fall back to GCS and never start. +// A single-component redeploy has to apply what deploy ate-system does: the +// same workloads, with the kind patches on kind, and the same storage settings +// from storageConfigPath. Applying the base storage settings on kind would +// point the object-store plugins at GCS. func TestComponentManifestsMatchSystemBundle(t *testing.T) { for _, tc := range []struct { router string @@ -227,5 +228,20 @@ func TestComponentManifestsMatchSystemBundle(t *testing.T) { t.Errorf("router=%s kind=%v: %s renders a different %s/%s than the system bundle (-bundle +standalone):\n%s", tc.router, tc.kind, component.path, component.kind, component.name, diff) } } + storage, err := kube.LoadPath(e.storageConfigPath()) + if err != nil { + t.Fatalf("LoadPath(%s): %v", e.storageConfigPath(), err) + } + if len(storage) == 0 { + t.Fatalf("%s holds no objects", e.storageConfigPath()) + } + for _, got := range storage { + want := findObject(bundle, got.GetKind(), got.GetName()) + if want == nil { + t.Errorf("router=%s kind=%v: the system bundle has no %s", tc.router, tc.kind, kube.Describe(got)) + } else if diff := cmp.Diff(want.Object, got.Object); diff != "" { + t.Errorf("router=%s kind=%v: %s holds a different %s than the system bundle (-bundle +standalone):\n%s", tc.router, tc.kind, e.storageConfigPath(), kube.Describe(got), diff) + } + } } } diff --git a/cmd/ate-setup/internal/steps/overlay.go b/cmd/ate-setup/internal/steps/overlay.go index 8a2d6d3e36..8f7852a1b9 100644 --- a/cmd/ate-setup/internal/steps/overlay.go +++ b/cmd/ate-setup/internal/steps/overlay.go @@ -545,6 +545,22 @@ func (e *Env) applyOtelConfig(ctx context.Context) error { return e.Kube.ApplyPath(ctx, e.otelConfigPath()) } +// storageConfigPath returns the environment's ate-storage-config ConfigMap, +// with the ate-storage-credentials Secret on kind. The objectstore-plugin +// sidecars of ate-api-server and atelet read both via envFrom. As with +// otelConfigPath, the base file on a kind cluster would point them at GCS. +func (e *Env) storageConfigPath() string { + if e.Cfg.Kind { + return e.Cfg.Manifest("kind", "ate-storage-config.yaml") + } + return e.Cfg.Manifest("ate-storage-config.yaml") +} + +// applyStorageConfig applies the environment's storage settings. +func (e *Env) applyStorageConfig(ctx context.Context) error { + return e.Kube.ApplyPath(ctx, e.storageConfigPath()) +} + // otelConfigMap is the ConfigMap every control plane component reads its // telemetry settings from through envFrom. const otelConfigMap = "ate-otel-config" diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index a626532e64..599c3db73c 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "maps" "os" "reflect" "slices" @@ -30,6 +31,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" dynamicfake "k8s.io/client-go/dynamic/fake" @@ -594,9 +596,8 @@ func dataPlaneWorkloads(t *testing.T, manifest []byte) (daemonSets map[string]ap // Under --fake-data-plane every path that applies atelet or ate-controller // carries the fake data plane: fake-atelet in atelet's DaemonSet, with the -// storage settings the overlay gives atelet's objectstore-plugin, whose -// writes it takes over, ate-controller at zero replicas, and fake-workersync -// beside them. +// storage settings of atelet's objectstore-plugin, whose writes it takes +// over, ate-controller at zero replicas, and fake-workersync beside them. func TestRenderFakeDataPlane(t *testing.T) { root := repoRoot(t) const ateletDS = "atelet" @@ -606,8 +607,8 @@ func TestRenderFakeDataPlane(t *testing.T) { path func(e *Env) string wantAtelet bool wantController bool - // wantStorage is the ATE_STORAGE_BACKEND the overlay gives - // atelet's objectstore-plugin. + // wantStorage is the ATE_STORAGE_BACKEND atelet's + // objectstore-plugin reads. wantStorage string }{ { @@ -702,7 +703,7 @@ func TestRenderFakeDataPlane(t *testing.T) { t.Errorf("atelet container image = %q, want %q", c.Image, want) } // fake-atelet runs with its defaults, tuned only through the - // ConfigMap a benchmark writes, beside atelet's own envFrom. + // ConfigMap a benchmark writes, beside the sidecar's envFrom. if len(c.Args) != 0 { t.Errorf("fake-atelet args = %v, want none", c.Args) } @@ -718,7 +719,7 @@ func TestRenderFakeDataPlane(t *testing.T) { if !slices.ContainsFunc(c.EnvFrom, func(src corev1.EnvFromSource) bool { return src.ConfigMapRef != nil && src.ConfigMapRef.Name == otelConfigMap }) { - t.Errorf("fake-atelet envFrom = %+v, lost atelet's %s", c.EnvFrom, otelConfigMap) + t.Errorf("fake-atelet envFrom = %+v, lost %s", c.EnvFrom, otelConfigMap) } for _, p := range c.Ports { if p.HostPort != 0 { @@ -735,10 +736,7 @@ func TestRenderFakeDataPlane(t *testing.T) { t.Errorf("fake-atelet probe %s targets port %q, which the container does not declare", probe.HTTPGet.Path, probe.HTTPGet.Port.StrVal) } } - env := map[string]string{} - for _, v := range c.Env { - env[v.Name] = v.Value - } + env := storageEnv(storageSettings(t, e), c) if env["ATE_STORAGE_BACKEND"] != tc.wantStorage { t.Errorf("fake-atelet ATE_STORAGE_BACKEND = %q, want objectstore-plugin's %q", env["ATE_STORAGE_BACKEND"], tc.wantStorage) } @@ -1511,3 +1509,147 @@ func TestRemoveFakeWorkersync(t *testing.T) { }) } } + +// storageSettings returns the keys of each ConfigMap and Secret in e's +// storage settings, by kind and name. +func storageSettings(t *testing.T, e *Env) map[string]map[string]string { + t.Helper() + objs, err := kube.LoadPath(e.storageConfigPath()) + if err != nil { + t.Fatalf("LoadPath(%s): %v", e.storageConfigPath(), err) + } + settings := map[string]map[string]string{} + for _, obj := range objs { + field := "data" + if obj.GetKind() == "Secret" { + field = "stringData" + } + data, _, err := unstructured.NestedStringMap(obj.Object, field) + if err != nil { + t.Fatalf("%s: %v", kube.Describe(obj), err) + } + settings[obj.GetKind()+"/"+obj.GetName()] = data + } + return settings +} + +// storageEnv returns the storage settings c runs with: its envFrom sources +// resolved from settings, then its env, which overrides them. +func storageEnv(settings map[string]map[string]string, c corev1.Container) map[string]string { + env := map[string]string{} + for _, src := range c.EnvFrom { + switch { + case src.ConfigMapRef != nil: + maps.Copy(env, settings["ConfigMap/"+src.ConfigMapRef.Name]) + case src.SecretRef != nil: + maps.Copy(env, settings["Secret/"+src.SecretRef.Name]) + } + } + for _, v := range c.Env { + env[v.Name] = v.Value + } + maps.DeleteFunc(env, func(k, _ string) bool { return !isStorageEnv(k) }) + return env +} + +func isStorageEnv(name string) bool { + return name == "ATE_STORAGE_BACKEND" || strings.HasPrefix(name, "AWS_") +} + +// The objectstore-plugin sidecars read their storage settings from +// ate-storage-config, which an inline copy would override. ATE_STORAGE_BACKEND +// must not reach the main containers: ate-api-server and atelet exit if it is +// set. +func TestObjectstorePluginsReadStorageConfig(t *testing.T) { + for _, tc := range []struct { + kind bool + wantBackend string + }{ + {false, "gcs"}, + {true, "s3"}, + } { + e := &Env{Cfg: &config.Config{Root: repoRoot(t), Kind: tc.kind}} + manifest, err := e.render(e.Cfg.Path(SystemOverlay(e.Cfg))) + if err != nil { + t.Fatalf("kind=%v: render: %v", tc.kind, err) + } + settings := storageSettings(t, e) + daemonSets, deployments := dataPlaneWorkloads(t, manifest) + for name, pod := range map[string]corev1.PodSpec{ + "ate-api-server": deployments["ate-api-server"].Spec.Template.Spec, + "atelet": daemonSets["atelet"].Spec.Template.Spec, + } { + if len(pod.InitContainers) == 0 || pod.InitContainers[0].Name != "objectstore-plugin" { + t.Fatalf("kind=%v: %s has no objectstore-plugin sidecar", tc.kind, name) + } + sidecar := pod.InitContainers[0] + for _, v := range sidecar.Env { + if isStorageEnv(v.Name) { + t.Errorf("kind=%v: %s objectstore-plugin sets %s inline", tc.kind, name, v.Name) + } + } + // Only kind ships the Secret. + if !slices.ContainsFunc(sidecar.EnvFrom, func(src corev1.EnvFromSource) bool { + ref := src.SecretRef + return ref != nil && ref.Name == "ate-storage-credentials" && ref.Optional != nil && *ref.Optional + }) { + t.Errorf("kind=%v: %s objectstore-plugin envFrom = %+v, want the optional ate-storage-credentials Secret", tc.kind, name, sidecar.EnvFrom) + } + if got := storageEnv(settings, sidecar)["ATE_STORAGE_BACKEND"]; got != tc.wantBackend { + t.Errorf("kind=%v: %s objectstore-plugin ATE_STORAGE_BACKEND = %q, want %q", tc.kind, name, got, tc.wantBackend) + } + for _, c := range pod.Containers { + if _, ok := storageEnv(settings, c)["ATE_STORAGE_BACKEND"]; ok { + t.Errorf("kind=%v: %s container %s gets ATE_STORAGE_BACKEND", tc.kind, name, c.Name) + } + } + } + } +} + +// kind's rustfs and its bucket-init Job set the same keys inline, so they have +// to match what the sidecars read. +func TestRustfsMatchesKindStorageConfig(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: repoRoot(t), Kind: true}} + settings := storageSettings(t, e) + want := map[string]string{} + maps.Copy(want, settings["ConfigMap/ate-storage-config"]) + maps.Copy(want, settings["Secret/ate-storage-credentials"]) + + objs, err := kube.LoadPath(e.Cfg.Manifest("kind", "rustfs.yaml")) + if err != nil { + t.Fatalf("LoadPath: %v", err) + } + env := func(kind, name string) map[string]string { + obj := findObject(objs, kind, name) + if obj == nil { + t.Fatalf("rustfs.yaml has no %s/%s", kind, name) + } + var workload struct { + Spec struct { + Template corev1.PodTemplateSpec `json:"template"` + } `json:"spec"` + } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &workload); err != nil { + t.Fatalf("%s: %v", kube.Describe(obj), err) + } + got := map[string]string{} + for _, c := range workload.Spec.Template.Spec.Containers { + for _, v := range c.Env { + got[v.Name] = v.Value + } + } + return got + } + + job := env("Job", "rustfs-bucket-init") + for _, k := range []string{"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION", "AWS_ENDPOINT_URL"} { + if job[k] != want[k] { + t.Errorf("rustfs-bucket-init %s = %q, want %q", k, job[k], want[k]) + } + } + server := env("Deployment", "rustfs") + if server["RUSTFS_ACCESS_KEY"] != want["AWS_ACCESS_KEY_ID"] || server["RUSTFS_SECRET_KEY"] != want["AWS_SECRET_ACCESS_KEY"] { + t.Errorf("rustfs keys = %q/%q, want the ate-storage-credentials keys", server["RUSTFS_ACCESS_KEY"], server["RUSTFS_SECRET_KEY"]) + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 8336ab2659..abc0177fbd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -329,7 +329,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov * **Lifecycle Management**: The `ateworker` process invokes the sandbox runtime to checkpoint or restore processes within the physical pod boundaries — `runsc` for gVisor, or the Kata + Cloud Hypervisor stack for microVMs. (Note: the gVisor backend currently requires a `runsc` version with the `--allow-connected-on-save` flag to work around a bug in networking resumption during checkpointing.) - * **Storage Mover**: External snapshots move through an *object-store plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/objectstore-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` on the sidecar (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which, see `manifests/ate-install/atelet.yaml`, `ate-api-server.yaml`, and the kind overlay for an S3 example. The same `atelet` sidecar serves `AssetProvider`, which fetches every sandbox asset; `atelet` reads a `gs://` asset anonymously only when the plugin cannot serve it, which covers public buckets such as `gs://gvisor`. The plugin stages each download where `atelet` cannot read it, and copies it into `atelet`'s cache only if its sha256 matches; it refuses any URL under a snapshot location. `atelet` verifies the sha256 again and has no object storage credentials or settings of its own. + * **Storage Mover**: External snapshots move through an *object-store plugin*, a sidecar that serves the `objectstore.v1` gRPC API over a Unix socket in its pod. `atelet`'s sidecar (`NodeProvider`) uploads and fetches a snapshot's files; `ate-api-server`'s sidecar (`ControlProvider`) copies and deletes whole snapshots. Neither main process talks to object storage for snapshots. The in-tree plugin, `cmd/objectstore-plugin`, backs the API with GCS or S3; `ATE_STORAGE_BACKEND` (unset or `gcs`, or `s3` with the `AWS_*` variables) selects which. Both sidecars read these settings from the `ate-storage-config` ConfigMap, and static S3 keys from the optional `ate-storage-credentials` Secret; see `manifests/ate-install/ate-storage-config.yaml`, and `manifests/ate-install/kind/ate-storage-config.yaml` for an S3 example. The same `atelet` sidecar serves `AssetProvider`, which fetches every sandbox asset; `atelet` reads a `gs://` asset anonymously only when the plugin cannot serve it, which covers public buckets such as `gs://gvisor`. The plugin stages each download where `atelet` cannot read it, and copies it into `atelet`'s cache only if its sha256 matches; it refuses any URL under a snapshot location. `atelet` verifies the sha256 again and has no object storage credentials or settings of its own. * **Sandbox Asset Cache**: `atelet` keeps one node-local, content-addressed cache of the sandbox binaries every actor runs on and gives each actor hard links into it, prewarms each local worker's default runtime and keeps it linked under the worker's directory, and evicts to a byte budget without taking anything an actor or a worker still uses. See the [Sandbox Asset Cache](sandbox-asset-cache.md) page. diff --git a/internal/e2e/README.md b/internal/e2e/README.md index 2e075470b5..fb57cb744e 100644 --- a/internal/e2e/README.md +++ b/internal/e2e/README.md @@ -87,8 +87,8 @@ lane that forgets cannot silently skip every preview test; see ## GCS-only suites `internal/e2e/suites/snapshotcredentials` runs only against an install that -stores snapshots in GCS, which it reads from the objectstore-plugin sidecar on -ate-api-server, and skips on any other backend, such as the kind lane's +stores snapshots in GCS, which it reads from the `ate-storage-config` +ConfigMap, and skips on any other backend, such as the kind lane's S3-compatible service. A lane that is meant to be GCS sets `E2E_REQUIRE_GCS=true`, which turns that skip into a failure, so a lane misconfigured as S3 cannot pass by running nothing. diff --git a/internal/e2e/backend.go b/internal/e2e/backend.go index cb77758a03..347538cb40 100644 --- a/internal/e2e/backend.go +++ b/internal/e2e/backend.go @@ -28,7 +28,7 @@ const GCSRequiredEnv = "E2E_REQUIRE_GCS" func GCSRequired() bool { return os.Getenv(GCSRequiredEnv) == "true" } // RequireGCSBackend skips t unless backend, the install's snapshot backend -// as the objectstore-plugin reports it, is GCS. Under GCSRequired it fails +// as its ate-storage-config gives it, is GCS. Under GCSRequired it fails // instead. func RequireGCSBackend(t testing.TB, backend string) { t.Helper() diff --git a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go index 502690aa07..56073d56e1 100644 --- a/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go +++ b/internal/e2e/suites/snapshotcredentials/snapshotcredentials_test.go @@ -51,24 +51,17 @@ import ( const probeTemplate = "probe" // requireGCS skips, or fails under e2e.GCSRequired, unless the install under -// test stores snapshots in GCS. The backend is the objectstore-plugin -// sidecar's ATE_STORAGE_BACKEND on ate-api-server. +// test stores snapshots in GCS. The backend is ATE_STORAGE_BACKEND in the +// ate-storage-config ConfigMap the objectstore-plugin sidecars read. func requireGCS(t *testing.T, ctx context.Context, clients *e2e.Clients) { t.Helper() - deploy, err := clients.K8s.AppsV1().Deployments(e2e.SystemNamespace()).Get(ctx, e2e.ResourceName("ate-api-server"), metav1.GetOptions{}) + cm, err := clients.K8s.CoreV1().ConfigMaps(e2e.SystemNamespace()).Get(ctx, e2e.ResourceName("ate-storage-config"), metav1.GetOptions{}) if err != nil { - t.Fatalf("getting the ate-api-server Deployment: %v", err) + t.Fatalf("getting the ate-storage-config ConfigMap: %v", err) } - backend := "gcs" - for _, c := range deploy.Spec.Template.Spec.InitContainers { - if c.Name != "objectstore-plugin" { - continue - } - for _, env := range c.Env { - if env.Name == "ATE_STORAGE_BACKEND" && env.Value != "" { - backend = env.Value - } - } + backend := cm.Data["ATE_STORAGE_BACKEND"] + if backend == "" { + backend = "gcs" } e2e.RequireGCSBackend(t, backend) } diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 07b1f6ff3c..d05b8f9091 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -126,15 +126,6 @@ spec: drop: - ALL env: - # The sidecar, not ate-api-server, holds the object storage settings: - # it copies external snapshots for tags and deletes the ones nothing - # refers to any more, so it needs the same backend atelet's sidecar - # reads and writes them through. Unset or "gcs" selects GCS with - # ambient credentials; "s3" with the AWS_* variables selects S3 (see - # the kind overlay). ate-api-server exits if ATE_STORAGE_BACKEND is - # set on it. - - name: ATE_STORAGE_BACKEND - value: "gcs" - name: ATE_API_ACTOR_JWT_ISSUER valueFrom: configMapKeyRef: @@ -151,9 +142,17 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # Object storage settings; see + # manifests/ate-install/ate-storage-config.yaml. ate-api-server exits if + # ATE_STORAGE_BACKEND is set on it. envFrom: - configMapRef: name: ate-otel-config + - configMapRef: + name: ate-storage-config + - secretRef: + name: ate-storage-credentials + optional: true volumeMounts: - name: objectstore-plugin mountPath: /run/objectstore-plugin diff --git a/manifests/ate-install/ate-storage-config.yaml b/manifests/ate-install/ate-storage-config.yaml new file mode 100644 index 0000000000..69b64ff308 --- /dev/null +++ b/manifests/ate-install/ate-storage-config.yaml @@ -0,0 +1,34 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Object storage settings for the objectstore-plugin sidecars of ate-api-server +# and atelet, which read them via envFrom. Both must use the same store: +# ate-api-server's sidecar copies and deletes the snapshots atelet's uploads. +# ATE_STORAGE_BACKEND "s3" with the AWS_* variables selects S3. Any other value +# selects GCS. +# +# This file holds the GCS settings. The kind overlay supplies a ConfigMap of the +# same name for rustfs, plus the optional ate-storage-credentials Secret; see +# manifests/ate-install/kind/ate-storage-config.yaml. +# +# NOTE: editing this ConfigMap does not restart the pods that consume it. Follow +# a change with `kubectl rollout restart` on deployment/ate-api-server and +# daemonset/atelet, so both sidecars use the same settings. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-storage-config + namespace: ate-system +data: + ATE_STORAGE_BACKEND: gcs diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index 8836ff8535..b780a23f88 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -195,13 +195,6 @@ spec: cpu: 50m memory: 128Mi env: - # Object storage settings for snapshot transfers and sandbox assets. - # Unset or "gcs" selects GCS with ambient credentials; "s3" with the - # AWS_* variables selects S3 (see the kind overlay). atelet has no - # storage settings of its own and exits if ATE_STORAGE_BACKEND is set - # on it. - - name: ATE_STORAGE_BACKEND - value: "gcs" # On GCS each transfer runs on a token scoped to its actor, from the # credential exchange the control sidecar serves on the ate-api-server # Service. The plugin derives the exchange's address and certificate @@ -211,9 +204,17 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # Object storage settings for snapshot transfers and sandbox assets; + # see manifests/ate-install/ate-storage-config.yaml. atelet exits if + # ATE_STORAGE_BACKEND is set on it. envFrom: - configMapRef: name: ate-otel-config + - configMapRef: + name: ate-storage-config + - secretRef: + name: ate-storage-credentials + optional: true volumeMounts: # HostToContainer so we also see mounts made under /var/lib/ate after # the pod starts, e.g. a snapshots disk or CSI volumes. diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index e1519c964a..a1ad911731 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -16,8 +16,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization # ate-otel-config.yaml carries the OTLP settings every component above consumes -# via envFrom. The kind overlay lists its own copy of the same ConfigMap name -# instead of building on this directory, so the two never collide. +# via envFrom, and ate-storage-config.yaml the objectstore-plugin sidecars' +# storage settings. The kind overlay lists its own copies instead of building on +# this directory, so they never collide. # # pod-certificate-controller.yaml is deliberately absent. The installers # (hack/install-ate.sh and cmd/ate-setup) apply it on their own before this @@ -35,4 +36,5 @@ resources: - ../atenet-router.yaml - ../atenet-router-monitoring.yaml - ../ate-otel-config.yaml + - ../ate-storage-config.yaml - ../ate-system-networkpolicies.yaml diff --git a/manifests/ate-install/components/fake-data-plane/kustomization.yaml b/manifests/ate-install/components/fake-data-plane/kustomization.yaml index dce35f26f9..a7d8e25e78 100644 --- a/manifests/ate-install/components/fake-data-plane/kustomization.yaml +++ b/manifests/ate-install/components/fake-data-plane/kustomization.yaml @@ -29,8 +29,8 @@ patches: # fake-atelet runs in atelet's DaemonSet, so it keeps atelet's name, labels, # service account (whose pod certificate carries atelet's SPIFFE ID), and # node selector. It writes snapshot placeholders itself, in place of the - # objectstore-plugin sidecar, so it takes that sidecar's env, the storage - # settings each overlay gives it, and the placeholders land where atelet's + # objectstore-plugin sidecar, so it takes that sidecar's env and envFrom, + # including ate-storage-config, and the placeholders land where atelet's # snapshots would. Everything atelet needs from the host goes, and so do its # scrape annotations, since fake-atelet serves no metrics. # ate.dev/fake-atelet is how fake-workersync finds the pods. @@ -52,6 +52,9 @@ patches: - op: copy from: /spec/template/spec/initContainers/0/env path: /spec/template/spec/containers/0/env + - op: copy + from: /spec/template/spec/initContainers/0/envFrom + path: /spec/template/spec/containers/0/envFrom - op: remove path: /spec/template/spec/initContainers - op: replace diff --git a/manifests/ate-install/kind/ate-api-server/kustomization.yaml b/manifests/ate-install/kind/ate-api-server/kustomization.yaml index c362bb6b00..38d172c8e5 100644 --- a/manifests/ate-install/kind/ate-api-server/kustomization.yaml +++ b/manifests/ate-install/kind/ate-api-server/kustomization.yaml @@ -24,10 +24,6 @@ resources: # patch rather than a key in ate-otel-config.yaml because that ConfigMap is # shared by every component via envFrom -- putting always_on there would pin the # router and the control plane to 100% too, undoing their per-component ratios. -# -# The storage settings point ate-api-server's object-store plugin at the same -# rustfs the ../atelet overlay points atelet's at: both ends of an external -# snapshot's life have to agree on where it lives. patches: - patch: |- apiVersion: apps/v1 @@ -43,19 +39,3 @@ patches: env: - name: OTEL_TRACES_SAMPLER value: parentbased_always_on - initContainers: - - name: objectstore-plugin - env: - - name: ATE_STORAGE_BACKEND - value: s3 - - name: AWS_REGION - value: us-east-1 - - name: AWS_ENDPOINT_URL - value: http://rustfs.ate-system.svc:9000 - - name: AWS_S3_USE_PATH_STYLE - value: "true" - # TODO: use a secret / identity management - - name: AWS_ACCESS_KEY_ID - value: rustfsadmin - - name: AWS_SECRET_ACCESS_KEY - value: rustfsadmin diff --git a/manifests/ate-install/kind/ate-storage-config.yaml b/manifests/ate-install/kind/ate-storage-config.yaml new file mode 100644 index 0000000000..fa9ca04082 --- /dev/null +++ b/manifests/ate-install/kind/ate-storage-config.yaml @@ -0,0 +1,38 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Kind replacement for manifests/ate-install/ate-storage-config.yaml: same name, +# so the overlay lists this file instead of the base one. Points both +# objectstore-plugin sidecars at the in-cluster rustfs from rustfs.yaml, with +# its static keys in the ate-storage-credentials Secret. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-storage-config + namespace: ate-system +data: + ATE_STORAGE_BACKEND: s3 + AWS_REGION: us-east-1 + AWS_ENDPOINT_URL: http://rustfs.ate-system.svc:9000 + AWS_S3_USE_PATH_STYLE: "true" +--- +apiVersion: v1 +kind: Secret +metadata: + name: ate-storage-credentials + namespace: ate-system +type: Opaque +stringData: + AWS_ACCESS_KEY_ID: rustfsadmin + AWS_SECRET_ACCESS_KEY: rustfsadmin diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index 0616973c6b..ac8cb3dee9 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -56,22 +56,3 @@ patches: $patch: delete - name: image-credential-provider-config $patch: delete - # The object-store plugin transfers atelet's external snapshots and - # fetches its private sandbox assets (the micro-VM assets - # stage-to-rustfs.sh uploads), so it points at rustfs. - initContainers: - - name: objectstore-plugin - env: - - name: ATE_STORAGE_BACKEND - value: s3 - - name: AWS_REGION - value: us-east-1 - - name: AWS_ENDPOINT_URL - value: http://rustfs.ate-system.svc:9000 - - name: AWS_S3_USE_PATH_STYLE - value: "true" - # TODO: use a secret / identity management - - name: AWS_ACCESS_KEY_ID - value: rustfsadmin - - name: AWS_SECRET_ACCESS_KEY - value: rustfsadmin diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index c757f2362f..bed3f7263a 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -17,10 +17,12 @@ kind: Kustomization # ate-otel-config.yaml replaces the base ../ate-otel-config.yaml (same ConfigMap # name) so every component's `envFrom` resolves to the in-cluster collector -# instead of the GKE one. It is listed here rather than in ./atelet or -# ./ate-api-server because those overlays are also built standalone; listing it -# in the bundle and an overlay would be a duplicate resource. ate-setup applies -# the ConfigMap directly for the single-component redeploys. +# instead of the GKE one. ate-storage-config.yaml likewise replaces the base +# copy and points the objectstore-plugin sidecars at rustfs. Both are listed +# here rather than in ./atelet or ./ate-api-server because those overlays are +# also built standalone; listing them in the bundle and an overlay would be a +# duplicate resource. ate-setup applies them directly for the single-component +# redeploys. # # pod-certificate-controller.yaml is deliberately absent, as in ../base: the # installers (hack/install-ate.sh and cmd/ate-setup) apply and delete it on @@ -34,6 +36,7 @@ resources: - ../atenet-router.yaml - ../ate-system-networkpolicies.yaml - ate-otel-config.yaml + - ate-storage-config.yaml - rustfs.yaml - ./otel-collector.yaml - ./prometheus.yaml diff --git a/manifests/ate-install/kind/rustfs.yaml b/manifests/ate-install/kind/rustfs.yaml index b37f52b9ce..3dbe970c2a 100644 --- a/manifests/ate-install/kind/rustfs.yaml +++ b/manifests/ate-install/kind/rustfs.yaml @@ -103,6 +103,9 @@ spec: containers: - name: create-bucket image: amazon/aws-cli:2.38.5@sha256:c8156a0e74e13929a42d7e60f0d4e14e7d28d980cd21822505d4bff80e5460c1 + # Same values as ate-storage-config.yaml. A Job's pod template is + # immutable, so reading them from there would fail the next + # `deploy ate-system` on a cluster that already ran this Job. env: - name: AWS_ACCESS_KEY_ID value: rustfsadmin @@ -133,7 +136,7 @@ spec: exit 1 --- # ../ate-system-networkpolicies.yaml denies ingress to every ate-system pod by -# default. rustfs's S3 API is reached by the snapshot-plugin sidecars of +# default. rustfs's S3 API is reached by the objectstore-plugin sidecars of # ate-api-server and atelet and by the bucket-init Job; its console is not # admitted (use a port-forward). apiVersion: networking.k8s.io/v1