From 69d07e0feb5789873780a31cb8e24b81dbfe969f Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sat, 10 Oct 2026 20:31:10 -0400 Subject: [PATCH 1/2] ate-setup: settings for the router's route timeout and parked budget --atenet-route-timeout and --atenet-parked-request-budget set the two router flags at install time; unset keeps the router defaults. A larger budget grows terminationGracePeriodSeconds by the same amount so parked requests finish during a rollout. Envoy dataplane only. applyWithAPIServerEnvHash now takes the decoded objects, so the system bundle is decoded once, the router flags are set, and the api-server env hash is stamped before the one apply. --- cmd/ate-setup/commands.md | 2 + cmd/ate-setup/internal/config/config.go | 4 + cmd/ate-setup/internal/config/config_test.go | 2 + cmd/ate-setup/internal/config/load.go | 26 +++- cmd/ate-setup/internal/config/load_test.go | 55 ++++++++ cmd/ate-setup/internal/config/setting.go | 11 ++ cmd/ate-setup/internal/steps/apienv.go | 14 +-- cmd/ate-setup/internal/steps/deploy.go | 24 +++- cmd/ate-setup/internal/steps/router.go | 124 ++++++++++++++++++ cmd/ate-setup/internal/steps/router_test.go | 125 +++++++++++++++++++ docs/examples/substrate-install.yaml | 4 + docs/operator-install.md | 2 + docs/request-parking.md | 11 ++ manifests/ate-install/atenet-router.yaml | 2 + 14 files changed, 389 insertions(+), 17 deletions(-) create mode 100644 cmd/ate-setup/internal/steps/router.go create mode 100644 cmd/ate-setup/internal/steps/router_test.go diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index c47ce3da89..4b2c6ef3ec 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -25,6 +25,8 @@ a pre-scan pass, so they may appear anywhere on its command line. |---|---|---| | `--kind` | `hack/install-ate-kind.sh`, or `ATE_INSTALL_KIND=true` | Kind overlays, the local registry, and host-architecture image builds | | `--atenet-dataplane envoy\|agentgateway` | `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) | +| `--atenet-route-timeout DURATION` | `ATE_ATENET_ROUTE_TIMEOUT=DURATION` | New. The router's `--route-timeout` (default `5m`). Envoy dataplane only | +| `--atenet-parked-request-budget DURATION` | `ATE_ATENET_PARKED_REQUEST_BUDGET=DURATION` | New. The router's `--parked-request-budget` (default `5s`); the router pod's `terminationGracePeriodSeconds` grows with it. Envoy dataplane only | | `--rollout-timeout DURATION` | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). Unlike the shell flag it also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | | `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | | `--cluster-size size0\|size10` | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index ab39897cf8..cb009626f0 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -158,6 +158,10 @@ type Config struct { // Router selects the atenet router dataplane. Router string + // RouterRouteTimeout and RouterParkedRequestBudget are the router's + // --route-timeout and --parked-request-budget; zero keeps the router default. + RouterRouteTimeout time.Duration + RouterParkedRequestBudget time.Duration // The read/write and owner connections can use different login identities. // With one configured connection, both pools use it. Both empty selects // bundled PostgreSQL. diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 99ae080bbd..5bcdb4e5aa 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -60,6 +60,8 @@ func loadEnv(t *testing.T) { "ATE_API_POSTGRES_SCHEMA", "ATE_API_POSTGRES_SERVER_CA_FILE", "ATE_ATENET_DATAPLANE", + "ATE_ATENET_PARKED_REQUEST_BUDGET", + "ATE_ATENET_ROUTE_TIMEOUT", "ATE_CREDENTIAL_PROVIDER", "ATE_IMAGE_REPO", "ATE_IMAGE_TAG", diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index a85f7d64d5..371a1a08a9 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -244,11 +244,13 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro Repo: strings.TrimSuffix(r.String("images.repo"), "/"), Tag: r.String("images.tag"), }, - Router: r.String("atenet.dataplane"), - PostgresSchema: r.String("ateapi.postgres.schema"), - PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"), - PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"), - PostgresStorageClass: r.String("ateapi.postgres.storageClass"), + Router: r.String("atenet.dataplane"), + RouterRouteTimeout: r.Duration("atenet.router.routeTimeout"), + RouterParkedRequestBudget: r.Duration("atenet.router.parkedRequestBudget"), + PostgresSchema: r.String("ateapi.postgres.schema"), + PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"), + PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"), + PostgresStorageClass: r.String("ateapi.postgres.storageClass"), PostgresReadWriteConnectionString: readWriteConnectionString, PostgresOwnerConnectionString: ownerConnectionString, @@ -399,6 +401,20 @@ func validateResolved(cfg *Config, r *Resolved) error { return &InvalidError{Value: dataplane, Want: RouterEnvoy + " or " + RouterAgentgateway} } + // Only the envoy dataplane runs the atenet-router container the flags go on. + for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget"} { + v, _ := r.Value(key) + if v.Raw == "" { + continue + } + if r.Duration(key) <= 0 { + return &InvalidError{Value: v, Want: "a positive duration"} + } + if cfg.Router != RouterEnvoy { + return &ConflictError{A: v, B: dataplane, Why: "the router flags need the envoy dataplane"} + } + } + if ipType, _ := r.Value("ateapi.postgres.cloudsql.ipType"); ipType.Raw != "" { switch ipType.Raw { case CloudSQLIPTypePrivate, CloudSQLIPTypePublic, CloudSQLIPTypePSC: diff --git a/cmd/ate-setup/internal/config/load_test.go b/cmd/ate-setup/internal/config/load_test.go index b48f8b3ebe..6b7b8b190b 100644 --- a/cmd/ate-setup/internal/config/load_test.go +++ b/cmd/ate-setup/internal/config/load_test.go @@ -20,6 +20,7 @@ import ( "runtime" "strings" "testing" + "time" ) // Under --kind the ambient environment is ignored, but a value the caller named @@ -224,3 +225,57 @@ func errPath(err error) string { } return rest } + +func TestLoadFlagsRouterSettings(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "30m") + t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m") + + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + cfg, err := LoadFlags(fs, LoadOptions{}) + if err != nil { + t.Fatalf("LoadFlags() error = %v", err) + } + if cfg.RouterRouteTimeout != 30*time.Minute { + t.Errorf("RouterRouteTimeout = %s, want 30m", cfg.RouterRouteTimeout) + } + if cfg.RouterParkedRequestBudget != 2*time.Minute { + t.Errorf("RouterParkedRequestBudget = %s, want 2m", cfg.RouterParkedRequestBudget) + } +} + +func TestLoadFlagsRouterSettingsNeedEnvoy(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway) + t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m") + + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + _, err = LoadFlags(fs, LoadOptions{}) + if err == nil { + t.Fatal("LoadFlags() = nil error, want a dataplane conflict") + } + for _, want := range []string{"ATE_ATENET_PARKED_REQUEST_BUDGET", "ATE_ATENET_DATAPLANE"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name %s", err, want) + } + } +} + +func TestLoadFlagsRouterSettingsMustBePositive(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "0s") + + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), "positive duration") { + t.Errorf("LoadFlags() error = %v, want a positive-duration error", err) + } +} diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 389da4d55c..834a771149 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -163,6 +163,17 @@ var Registry = []Setting{ Default: RouterEnvoy, Usage: "Atenet ingress and egress dataplane: envoy or agentgateway", }, + { + // Set on the envoy dataplane's atenet-router container; see steps.configureAtenetRouter. + Key: "atenet.router.routeTimeout", Env: "ATE_ATENET_ROUTE_TIMEOUT", Flag: "atenet-route-timeout", + Kind: KindDuration, + Usage: "The router's --route-timeout; unset keeps the router default (envoy dataplane only)", + }, + { + Key: "atenet.router.parkedRequestBudget", Env: "ATE_ATENET_PARKED_REQUEST_BUDGET", Flag: "atenet-parked-request-budget", + Kind: KindDuration, + Usage: "The router's --parked-request-budget; unset keeps the router default; the router pod's grace period grows with it (envoy dataplane only)", + }, { Key: "atenet.egress.additionalExtprocService", Env: "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", Flag: "experimental-additional-egress-extproc-service", Kind: KindString, diff --git a/cmd/ate-setup/internal/steps/apienv.go b/cmd/ate-setup/internal/steps/apienv.go index 481f72de1e..0cd955f53a 100644 --- a/cmd/ate-setup/internal/steps/apienv.go +++ b/cmd/ate-setup/internal/steps/apienv.go @@ -336,15 +336,11 @@ func (e *Env) annotateAPIServerEnvHash(ctx context.Context) error { return e.Kube.PatchDeployment(ctx, e.Namespace(), "ate-api-server", []byte(patch)) } -// applyWithAPIServerEnvHash applies a manifest that carries the ate-api-server -// Deployment, with the digest of its current environment on the pod template. -// A new image and a new environment are then one rollout, and a fresh install -// starts out stamped. -func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, manifest []byte) error { - objs, err := kube.DecodeManifestBytes(manifest) - if err != nil { - return err - } +// applyWithAPIServerEnvHash applies a decoded manifest that carries the +// ate-api-server Deployment, with the digest of its current environment on the +// pod template. A new image and a new environment are then one rollout, and a +// fresh install starts out stamped. +func (e *Env) applyWithAPIServerEnvHash(ctx context.Context, objs []*unstructured.Unstructured) error { hash, err := e.apiServerEnvHash(ctx) if err != nil { return err diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index e235e96b0a..2353c26362 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -148,7 +148,14 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.removeFakeWorkersync(ctx); err != nil { return err } - if err := e.applyWithAPIServerEnvHash(ctx, manifests); err != nil { + objs, err := kube.DecodeManifestBytes(manifests) + if err != nil { + return err + } + if err := e.configureAtenetRouter(objs); err != nil { + return err + } + if err := e.applyWithAPIServerEnvHash(ctx, objs); err != nil { return err } @@ -332,7 +339,11 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error { if err != nil { return err } - if err := e.applyWithAPIServerEnvHash(ctx, manifest); err != nil { + objs, err := kube.DecodeManifestBytes(manifest) + if err != nil { + return err + } + if err := e.applyWithAPIServerEnvHash(ctx, objs); err != nil { return err } // After the manifest, which resets the pod template to the sidecar-free base. @@ -558,7 +569,14 @@ func (e *Env) DeployAtenet(ctx context.Context) error { if err != nil { return err } - if err := e.Kube.ApplyBytes(ctx, routerManifest); err != nil { + routerObjs, err := kube.DecodeManifestBytes(routerManifest) + if err != nil { + return err + } + if err := e.configureAtenetRouter(routerObjs); err != nil { + return err + } + if err := e.Kube.Apply(ctx, routerObjs); err != nil { return err } if err := e.EnsureEgressMITMCAPoolSecret(ctx); err != nil { diff --git a/cmd/ate-setup/internal/steps/router.go b/cmd/ate-setup/internal/steps/router.go new file mode 100644 index 0000000000..5e4ed1b869 --- /dev/null +++ b/cmd/ate-setup/internal/steps/router.go @@ -0,0 +1,124 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "fmt" + "math" + "strings" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" + "go.ate.dev/substrate/cmd/ate-setup/internal/log" +) + +// manifestParkedRequestBudget is the budget the manifest's +// terminationGracePeriodSeconds is sized for: the router default. +const manifestParkedRequestBudget = 5 * time.Second + +// configureAtenetRouter sets --route-timeout and --parked-request-budget on +// the atenet-router container and grows terminationGracePeriodSeconds with the +// budget, since the router's drain deadline follows it. A no-op when unset. +func (e *Env) configureAtenetRouter(objs []*unstructured.Unstructured) error { + routeTimeout, budget := e.Cfg.RouterRouteTimeout, e.Cfg.RouterParkedRequestBudget + if routeTimeout <= 0 && budget <= 0 { + return nil + } + var dep *unstructured.Unstructured + for _, obj := range objs { + if obj.GetKind() == "Deployment" && obj.GetNamespace() == e.Namespace() && obj.GetName() == "atenet-router" { + dep = obj + break + } + } + if dep == nil { + return fmt.Errorf("the router manifest has no deployment/atenet-router in %s", e.Namespace()) + } + + containers, _, err := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") + if err != nil { + return fmt.Errorf("%s: %w", kube.Describe(dep), err) + } + idx := -1 + for i, c := range containers { + if m, ok := c.(map[string]any); ok && m["name"] == "atenet-router" { + idx = i + break + } + } + if idx < 0 { + return fmt.Errorf("%s has no atenet-router container for the router flags", kube.Describe(dep)) + } + container := containers[idx].(map[string]any) + args, _, err := unstructured.NestedSlice(container, "args") + if err != nil { + return fmt.Errorf("%s: %w", kube.Describe(dep), err) + } + + if routeTimeout > 0 { + log.Infof("Setting the router's --route-timeout to %s", routeTimeout) + args = setFlag(args, "--route-timeout", routeTimeout.String()) + } + if budget > 0 { + log.Infof("Setting the router's --parked-request-budget to %s", budget) + args = setFlag(args, "--parked-request-budget", budget.String()) + if extra := budget - manifestParkedRequestBudget; extra > 0 { + grace, err := gracePeriod(dep) + if err != nil { + return err + } + grace += int64(math.Ceil(extra.Seconds())) + if err := unstructured.SetNestedField(dep.Object, grace, "spec", "template", "spec", "terminationGracePeriodSeconds"); err != nil { + return fmt.Errorf("%s: %w", kube.Describe(dep), err) + } + } + } + container["args"] = args + containers[idx] = container + return unstructured.SetNestedSlice(dep.Object, containers, "spec", "template", "spec", "containers") +} + +// setFlag replaces the --flag=value argument in args, or appends it. +func setFlag(args []any, flag, value string) []any { + arg := flag + "=" + value + for i, a := range args { + if s, ok := a.(string); ok && strings.HasPrefix(s, flag+"=") { + args[i] = arg + return args + } + } + return append(args, arg) +} + +// gracePeriod reads terminationGracePeriodSeconds; YAML decodes it as float64, JSON as int64. +func gracePeriod(dep *unstructured.Unstructured) (int64, error) { + raw, found, err := unstructured.NestedFieldNoCopy(dep.Object, "spec", "template", "spec", "terminationGracePeriodSeconds") + if err != nil { + return 0, fmt.Errorf("%s: %w", kube.Describe(dep), err) + } + if !found { + return 30, nil // the Kubernetes default + } + switch v := raw.(type) { + case int64: + return v, nil + case float64: + return int64(v), nil + default: + return 0, fmt.Errorf("%s: terminationGracePeriodSeconds is %T, want a number", kube.Describe(dep), raw) + } +} diff --git a/cmd/ate-setup/internal/steps/router_test.go b/cmd/ate-setup/internal/steps/router_test.go new file mode 100644 index 0000000000..dc7e7ef396 --- /dev/null +++ b/cmd/ate-setup/internal/steps/router_test.go @@ -0,0 +1,125 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package steps + +import ( + "reflect" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "go.ate.dev/substrate/cmd/ate-setup/internal/config" + "go.ate.dev/substrate/cmd/ate-setup/internal/kube" +) + +func routerObjects(t *testing.T, cfg *config.Config) ([]*unstructured.Unstructured, *Env) { + t.Helper() + cfg.Root = repoRoot(t) + e := &Env{Cfg: cfg} + manifest, err := e.render(e.Cfg.Manifest("atenet-router.yaml")) + if err != nil { + t.Fatalf("render(atenet-router.yaml): %v", err) + } + objs, err := kube.DecodeManifestBytes(manifest) + if err != nil { + t.Fatalf("decoding the router manifest: %v", err) + } + return objs, e +} + +func routerPodSpec(t *testing.T, objs []*unstructured.Unstructured) ([]any, int64) { + t.Helper() + dep := findObject(objs, "Deployment", "atenet-router") + if dep == nil { + t.Fatal("manifest has no deployment/atenet-router") + } + grace, err := gracePeriod(dep) + if err != nil { + t.Fatal(err) + } + containers, _, _ := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") + for _, c := range containers { + if m := c.(map[string]any); m["name"] == "atenet-router" { + args, _, _ := unstructured.NestedSlice(m, "args") + return args, grace + } + } + t.Fatal("deployment/atenet-router has no atenet-router container") + return nil, 0 +} + +func flagValues(args []any, flag string) []string { + var out []string + for _, a := range args { + if s, ok := a.(string); ok && strings.HasPrefix(s, flag+"=") { + out = append(out, strings.TrimPrefix(s, flag+"=")) + } + } + return out +} + +func TestConfigureAtenetRouter(t *testing.T) { + objs, e := routerObjects(t, &config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute}) + _, grace := routerPodSpec(t, objs) + if err := e.configureAtenetRouter(objs); err != nil { + t.Fatalf("configureAtenetRouter: %v", err) + } + args, got := routerPodSpec(t, objs) + if v := flagValues(args, "--route-timeout"); len(v) != 1 || v[0] != "30m0s" { + t.Errorf("--route-timeout = %v, want [30m0s]", v) + } + if v := flagValues(args, "--parked-request-budget"); len(v) != 1 || v[0] != "2m0s" { + t.Errorf("--parked-request-budget = %v, want [2m0s]", v) + } + if want := grace + 115; got != want { + t.Errorf("terminationGracePeriodSeconds = %d, want %d", got, want) + } +} + +// Unset settings keep the manifest as checked in, which must not set the flags itself. +func TestConfigureAtenetRouterUnsetLeavesManifestAlone(t *testing.T) { + objs, e := routerObjects(t, &config.Config{}) + before, grace := routerPodSpec(t, objs) + for _, flag := range []string{"--route-timeout", "--parked-request-budget"} { + if v := flagValues(before, flag); len(v) != 0 { + t.Errorf("the manifest sets %s=%v; the settings expect it unset", flag, v) + } + } + if err := e.configureAtenetRouter(objs); err != nil { + t.Fatalf("configureAtenetRouter: %v", err) + } + after, got := routerPodSpec(t, objs) + if !reflect.DeepEqual(before, after) || got != grace { + t.Errorf("args or grace period changed with no setting set: %v %d, want %v %d", after, got, before, grace) + } +} + +// The agentgateway overlay replaces the atenet-router container. +func TestConfigureAtenetRouterRejectsMissingContainer(t *testing.T) { + dep := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{"namespace": NamespaceAteSystem, "name": "atenet-router"}, + "spec": map[string]any{"template": map[string]any{"spec": map[string]any{ + "containers": []any{map[string]any{"name": "agentgateway"}}, + }}}, + }} + e := &Env{Cfg: &config.Config{RouterRouteTimeout: time.Minute}} + if err := e.configureAtenetRouter([]*unstructured.Unstructured{dep}); err == nil { + t.Error("configureAtenetRouter succeeded without an atenet-router container, want an error") + } +} diff --git a/docs/examples/substrate-install.yaml b/docs/examples/substrate-install.yaml index e1077f6563..e0c880f261 100644 --- a/docs/examples/substrate-install.yaml +++ b/docs/examples/substrate-install.yaml @@ -39,6 +39,10 @@ rolloutTimeout: 1m0s # Ingress and egress dataplane: envoy or agentgateway. atenet: dataplane: envoy + # Router timeouts for agent workloads; unset keeps the router defaults (5s, 5m). Envoy only. + router: + parkedRequestBudget: 2m + routeTimeout: 30m # Prebuilt images. Both halves are required together: a tag with no repository # would be dropped, leaving a build from source that looks like the release the diff --git a/docs/operator-install.md b/docs/operator-install.md index 86b46f6d5b..51443c6ef6 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -294,6 +294,8 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `atenet.dataplane` | `ATE_ATENET_DATAPLANE` | `--atenet-dataplane` | `envoy` | (global) | | `atenet.egress.additionalExtprocService` | `ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE` | `--experimental-additional-egress-extproc-service` | — | (global) | | `atenet.egress.credentialProvider` | `ATE_CREDENTIAL_PROVIDER` | `--credential-provider` | — | (global) | +| `atenet.router.parkedRequestBudget` | `ATE_ATENET_PARKED_REQUEST_BUDGET` | `--atenet-parked-request-budget` | — | (global) | +| `atenet.router.routeTimeout` | `ATE_ATENET_ROUTE_TIMEOUT` | `--atenet-route-timeout` | — | (global) | | `clusterSize` | `ATE_INSTALL_CLUSTER_SIZE` | `--cluster-size` | `size0` | (global) | | `context` | `KUBECTL_CONTEXT` | `--context` | — | (global) | | `cordonControlPlane` | `ATE_INSTALL_CORDON_CONTROL_PLANE` | `--cordon-control-plane` | `false` | (global) | diff --git a/docs/request-parking.md b/docs/request-parking.md index 69ab7486fb..9f48bff13e 100644 --- a/docs/request-parking.md +++ b/docs/request-parking.md @@ -149,6 +149,17 @@ so a parked request always gets its full budget and a normal verdict (routed The retry backoff deliberately has no cap and no attempt limit: the budget alone bounds the wait. +### Sizing for agent workloads + +The defaults suit short interactive turns. For agents that run long commands, +or pools that are often full, set the budget to the longest wait for a slot +you accept before a `503`, and the route timeout above the longest request an +actor serves. Both are install settings, `--atenet-parked-request-budget` and +`--atenet-route-timeout` (`ATE_ATENET_PARKED_REQUEST_BUDGET`, +`ATE_ATENET_ROUTE_TIMEOUT`); the installer grows the router's +`terminationGracePeriodSeconds` with the budget so parked requests still +finish during a rollout. + ## Observability **Metrics** (OpenTelemetry, meter `atenet-router`): diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index d9185f591c..ad0efd9d51 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -244,6 +244,8 @@ spec: # default of 5m, which suits an agent relaying a model completion: the # request stays open for the whole generation. Set --route-timeout # lower to cap how long one turn may hold a request open. + # ate-setup appends --route-timeout and --parked-request-budget here from + # its atenet.router.* settings and grows the grace period above with the budget. # # A turn still running after the drain route budget (10s) does not # survive a shutdown, and raising --route-timeout does not change that: From 932137ca2f3323ea0e0db9ab7b7a478c3f8066ea Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 13 Oct 2026 09:50:12 -0400 Subject: [PATCH 2/2] ate-setup: setting for the router's ext_proc circuit breaker ATE_ATENET_EXTPROC_MAX_REQUESTS (atenet.router.extprocMaxRequests, --atenet-extproc-max-requests) passes --extproc-max-requests to the router the way the route timeout and parked budget settings pass theirs. On a dense host 1536 synchronized execs overflowed the derived breaker of 2048 and Envoy answered 503 "reset reason: overflow"; the only way to raise it was a patch on the deployment. --- cmd/ate-setup/commands.md | 1 + cmd/ate-setup/internal/config/config.go | 3 + cmd/ate-setup/internal/config/config_test.go | 1 + cmd/ate-setup/internal/config/load.go | 14 ++++- cmd/ate-setup/internal/config/load_test.go | 65 +++++++++++++------- cmd/ate-setup/internal/config/setting.go | 5 ++ cmd/ate-setup/internal/steps/router.go | 16 +++-- cmd/ate-setup/internal/steps/router_test.go | 28 ++++++++- docs/examples/substrate-install.yaml | 6 +- docs/operator-install.md | 1 + docs/request-parking.md | 7 +++ manifests/ate-install/atenet-router.yaml | 5 +- 12 files changed, 116 insertions(+), 36 deletions(-) diff --git a/cmd/ate-setup/commands.md b/cmd/ate-setup/commands.md index 4b2c6ef3ec..549762e602 100644 --- a/cmd/ate-setup/commands.md +++ b/cmd/ate-setup/commands.md @@ -27,6 +27,7 @@ a pre-scan pass, so they may appear anywhere on its command line. | `--atenet-dataplane envoy\|agentgateway` | `--atenet-dataplane envoy\|agentgateway` | atenet ingress and egress dataplane (default `envoy`) | | `--atenet-route-timeout DURATION` | `ATE_ATENET_ROUTE_TIMEOUT=DURATION` | New. The router's `--route-timeout` (default `5m`). Envoy dataplane only | | `--atenet-parked-request-budget DURATION` | `ATE_ATENET_PARKED_REQUEST_BUDGET=DURATION` | New. The router's `--parked-request-budget` (default `5s`); the router pod's `terminationGracePeriodSeconds` grows with it. Envoy dataplane only | +| `--atenet-extproc-max-requests N` | `ATE_ATENET_EXTPROC_MAX_REQUESTS=N` | New. The router's `--extproc-max-requests`, the circuit breaker on Envoy's ext_proc cluster (default: twice `--parked-request-max`, at least `1024`). Envoy dataplane only | | `--rollout-timeout DURATION` | `--rollout-timeout DURATION` | Readiness timeout for workloads (default `60s`). Unlike the shell flag it also governs the podcertificate-controller and CSI waits, which stay at their 120s default until it is passed | | `--podcert-workers-per-signer N` | `--podcert-workers-per-signer N` | Concurrent workers per podcertificate-controller signer | | `--cluster-size size0\|size10` | `--cluster-size size0\|size10` | Footprint profile (default `size0`). `size10` assumes a dedicated PostgreSQL node: it resizes the bundled StatefulSet and its `postgresql.conf`, pins the apiserver's connection pool, and raises the podcertificate-controller's API rate limits. `ATE_INSTALL_CLUSTER_SIZE` when the flag is absent | diff --git a/cmd/ate-setup/internal/config/config.go b/cmd/ate-setup/internal/config/config.go index cb009626f0..b176b1adbc 100644 --- a/cmd/ate-setup/internal/config/config.go +++ b/cmd/ate-setup/internal/config/config.go @@ -162,6 +162,9 @@ type Config struct { // --route-timeout and --parked-request-budget; zero keeps the router default. RouterRouteTimeout time.Duration RouterParkedRequestBudget time.Duration + // RouterExtProcMaxRequests is the router's --extproc-max-requests; zero + // keeps the router's derivation from --parked-request-max. + RouterExtProcMaxRequests int // The read/write and owner connections can use different login identities. // With one configured connection, both pools use it. Both empty selects // bundled PostgreSQL. diff --git a/cmd/ate-setup/internal/config/config_test.go b/cmd/ate-setup/internal/config/config_test.go index 5bcdb4e5aa..48ccc5f4c3 100644 --- a/cmd/ate-setup/internal/config/config_test.go +++ b/cmd/ate-setup/internal/config/config_test.go @@ -60,6 +60,7 @@ func loadEnv(t *testing.T) { "ATE_API_POSTGRES_SCHEMA", "ATE_API_POSTGRES_SERVER_CA_FILE", "ATE_ATENET_DATAPLANE", + "ATE_ATENET_EXTPROC_MAX_REQUESTS", "ATE_ATENET_PARKED_REQUEST_BUDGET", "ATE_ATENET_ROUTE_TIMEOUT", "ATE_CREDENTIAL_PROVIDER", diff --git a/cmd/ate-setup/internal/config/load.go b/cmd/ate-setup/internal/config/load.go index 371a1a08a9..f91e3732bb 100644 --- a/cmd/ate-setup/internal/config/load.go +++ b/cmd/ate-setup/internal/config/load.go @@ -247,6 +247,7 @@ func buildConfig(root string, env map[string]string, r *Resolved) (*Config, erro Router: r.String("atenet.dataplane"), RouterRouteTimeout: r.Duration("atenet.router.routeTimeout"), RouterParkedRequestBudget: r.Duration("atenet.router.parkedRequestBudget"), + RouterExtProcMaxRequests: r.Int("atenet.router.extprocMaxRequests"), PostgresSchema: r.String("ateapi.postgres.schema"), PostgresPoolMaxConns: r.String("ateapi.postgres.poolMaxConns"), PostgresServerCAFile: r.String("ateapi.postgres.serverCAFile"), @@ -402,13 +403,20 @@ func validateResolved(cfg *Config, r *Resolved) error { } // Only the envoy dataplane runs the atenet-router container the flags go on. - for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget"} { + for _, key := range []string{"atenet.router.routeTimeout", "atenet.router.parkedRequestBudget", "atenet.router.extprocMaxRequests"} { v, _ := r.Value(key) if v.Raw == "" { continue } - if r.Duration(key) <= 0 { - return &InvalidError{Value: v, Want: "a positive duration"} + switch v.Setting.Kind { + case KindDuration: + if r.Duration(key) <= 0 { + return &InvalidError{Value: v, Want: "a positive duration"} + } + case KindInt: + if r.Int(key) <= 0 { + return &InvalidError{Value: v, Want: "a positive integer"} + } } if cfg.Router != RouterEnvoy { return &ConflictError{A: v, B: dataplane, Why: "the router flags need the envoy dataplane"} diff --git a/cmd/ate-setup/internal/config/load_test.go b/cmd/ate-setup/internal/config/load_test.go index 6b7b8b190b..05a44f2f3f 100644 --- a/cmd/ate-setup/internal/config/load_test.go +++ b/cmd/ate-setup/internal/config/load_test.go @@ -230,6 +230,7 @@ func TestLoadFlagsRouterSettings(t *testing.T) { loadEnv(t) t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "30m") t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m") + t.Setenv("ATE_ATENET_EXTPROC_MAX_REQUESTS", "4096") fs, err := optionsFlagSet(Options{}) if err != nil { @@ -245,37 +246,55 @@ func TestLoadFlagsRouterSettings(t *testing.T) { if cfg.RouterParkedRequestBudget != 2*time.Minute { t.Errorf("RouterParkedRequestBudget = %s, want 2m", cfg.RouterParkedRequestBudget) } + if cfg.RouterExtProcMaxRequests != 4096 { + t.Errorf("RouterExtProcMaxRequests = %d, want 4096", cfg.RouterExtProcMaxRequests) + } } func TestLoadFlagsRouterSettingsNeedEnvoy(t *testing.T) { - loadEnv(t) - t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway) - t.Setenv("ATE_ATENET_PARKED_REQUEST_BUDGET", "2m") + for env, value := range map[string]string{ + "ATE_ATENET_PARKED_REQUEST_BUDGET": "2m", + "ATE_ATENET_EXTPROC_MAX_REQUESTS": "4096", + } { + t.Run(env, func(t *testing.T) { + loadEnv(t) + t.Setenv("ATE_ATENET_DATAPLANE", RouterAgentgateway) + t.Setenv(env, value) - fs, err := optionsFlagSet(Options{}) - if err != nil { - t.Fatal(err) - } - _, err = LoadFlags(fs, LoadOptions{}) - if err == nil { - t.Fatal("LoadFlags() = nil error, want a dataplane conflict") - } - for _, want := range []string{"ATE_ATENET_PARKED_REQUEST_BUDGET", "ATE_ATENET_DATAPLANE"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("error %q does not name %s", err, want) - } + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + _, err = LoadFlags(fs, LoadOptions{}) + if err == nil { + t.Fatal("LoadFlags() = nil error, want a dataplane conflict") + } + for _, want := range []string{env, "ATE_ATENET_DATAPLANE"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name %s", err, want) + } + } + }) } } func TestLoadFlagsRouterSettingsMustBePositive(t *testing.T) { - loadEnv(t) - t.Setenv("ATE_ATENET_ROUTE_TIMEOUT", "0s") + for _, tc := range []struct{ env, value, want string }{ + {"ATE_ATENET_ROUTE_TIMEOUT", "0s", "positive duration"}, + {"ATE_ATENET_EXTPROC_MAX_REQUESTS", "0", "positive integer"}, + {"ATE_ATENET_EXTPROC_MAX_REQUESTS", "-1", "positive integer"}, + } { + t.Run(tc.env+"="+tc.value, func(t *testing.T) { + loadEnv(t) + t.Setenv(tc.env, tc.value) - fs, err := optionsFlagSet(Options{}) - if err != nil { - t.Fatal(err) - } - if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), "positive duration") { - t.Errorf("LoadFlags() error = %v, want a positive-duration error", err) + fs, err := optionsFlagSet(Options{}) + if err != nil { + t.Fatal(err) + } + if _, err := LoadFlags(fs, LoadOptions{}); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("LoadFlags() error = %v, want a %s error", err, tc.want) + } + }) } } diff --git a/cmd/ate-setup/internal/config/setting.go b/cmd/ate-setup/internal/config/setting.go index 834a771149..ceea4a1737 100644 --- a/cmd/ate-setup/internal/config/setting.go +++ b/cmd/ate-setup/internal/config/setting.go @@ -174,6 +174,11 @@ var Registry = []Setting{ Kind: KindDuration, Usage: "The router's --parked-request-budget; unset keeps the router default; the router pod's grace period grows with it (envoy dataplane only)", }, + { + Key: "atenet.router.extprocMaxRequests", Env: "ATE_ATENET_EXTPROC_MAX_REQUESTS", Flag: "atenet-extproc-max-requests", + Kind: KindInt, + Usage: "The router's --extproc-max-requests, the circuit breaker on Envoy's ext_proc cluster; unset keeps the router's derivation (envoy dataplane only)", + }, { Key: "atenet.egress.additionalExtprocService", Env: "ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE", Flag: "experimental-additional-egress-extproc-service", Kind: KindString, diff --git a/cmd/ate-setup/internal/steps/router.go b/cmd/ate-setup/internal/steps/router.go index 5e4ed1b869..86bad79ffe 100644 --- a/cmd/ate-setup/internal/steps/router.go +++ b/cmd/ate-setup/internal/steps/router.go @@ -17,6 +17,7 @@ package steps import ( "fmt" "math" + "strconv" "strings" "time" @@ -30,12 +31,13 @@ import ( // terminationGracePeriodSeconds is sized for: the router default. const manifestParkedRequestBudget = 5 * time.Second -// configureAtenetRouter sets --route-timeout and --parked-request-budget on -// the atenet-router container and grows terminationGracePeriodSeconds with the -// budget, since the router's drain deadline follows it. A no-op when unset. +// configureAtenetRouter sets --route-timeout, --parked-request-budget and +// --extproc-max-requests on the atenet-router container and grows +// terminationGracePeriodSeconds with the budget, since the router's drain +// deadline follows it. A no-op when unset. func (e *Env) configureAtenetRouter(objs []*unstructured.Unstructured) error { - routeTimeout, budget := e.Cfg.RouterRouteTimeout, e.Cfg.RouterParkedRequestBudget - if routeTimeout <= 0 && budget <= 0 { + routeTimeout, budget, maxRequests := e.Cfg.RouterRouteTimeout, e.Cfg.RouterParkedRequestBudget, e.Cfg.RouterExtProcMaxRequests + if routeTimeout <= 0 && budget <= 0 && maxRequests <= 0 { return nil } var dep *unstructured.Unstructured @@ -87,6 +89,10 @@ func (e *Env) configureAtenetRouter(objs []*unstructured.Unstructured) error { } } } + if maxRequests > 0 { + log.Infof("Setting the router's --extproc-max-requests to %d", maxRequests) + args = setFlag(args, "--extproc-max-requests", strconv.Itoa(maxRequests)) + } container["args"] = args containers[idx] = container return unstructured.SetNestedSlice(dep.Object, containers, "spec", "template", "spec", "containers") diff --git a/cmd/ate-setup/internal/steps/router_test.go b/cmd/ate-setup/internal/steps/router_test.go index dc7e7ef396..47efd104bd 100644 --- a/cmd/ate-setup/internal/steps/router_test.go +++ b/cmd/ate-setup/internal/steps/router_test.go @@ -73,7 +73,7 @@ func flagValues(args []any, flag string) []string { } func TestConfigureAtenetRouter(t *testing.T) { - objs, e := routerObjects(t, &config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute}) + objs, e := routerObjects(t, &config.Config{RouterRouteTimeout: 30 * time.Minute, RouterParkedRequestBudget: 2 * time.Minute, RouterExtProcMaxRequests: 4096}) _, grace := routerPodSpec(t, objs) if err := e.configureAtenetRouter(objs); err != nil { t.Fatalf("configureAtenetRouter: %v", err) @@ -85,16 +85,40 @@ func TestConfigureAtenetRouter(t *testing.T) { if v := flagValues(args, "--parked-request-budget"); len(v) != 1 || v[0] != "2m0s" { t.Errorf("--parked-request-budget = %v, want [2m0s]", v) } + if v := flagValues(args, "--extproc-max-requests"); len(v) != 1 || v[0] != "4096" { + t.Errorf("--extproc-max-requests = %v, want [4096]", v) + } if want := grace + 115; got != want { t.Errorf("terminationGracePeriodSeconds = %d, want %d", got, want) } } +// The breaker alone must still reach the container, with the grace period untouched. +func TestConfigureAtenetRouterExtProcMaxRequestsAlone(t *testing.T) { + objs, e := routerObjects(t, &config.Config{RouterExtProcMaxRequests: 4096}) + _, grace := routerPodSpec(t, objs) + if err := e.configureAtenetRouter(objs); err != nil { + t.Fatalf("configureAtenetRouter: %v", err) + } + args, got := routerPodSpec(t, objs) + if v := flagValues(args, "--extproc-max-requests"); len(v) != 1 || v[0] != "4096" { + t.Errorf("--extproc-max-requests = %v, want [4096]", v) + } + for _, flag := range []string{"--route-timeout", "--parked-request-budget"} { + if v := flagValues(args, flag); len(v) != 0 { + t.Errorf("%s = %v, want unset", flag, v) + } + } + if got != grace { + t.Errorf("terminationGracePeriodSeconds = %d, want %d unchanged", got, grace) + } +} + // Unset settings keep the manifest as checked in, which must not set the flags itself. func TestConfigureAtenetRouterUnsetLeavesManifestAlone(t *testing.T) { objs, e := routerObjects(t, &config.Config{}) before, grace := routerPodSpec(t, objs) - for _, flag := range []string{"--route-timeout", "--parked-request-budget"} { + for _, flag := range []string{"--route-timeout", "--parked-request-budget", "--extproc-max-requests"} { if v := flagValues(before, flag); len(v) != 0 { t.Errorf("the manifest sets %s=%v; the settings expect it unset", flag, v) } diff --git a/docs/examples/substrate-install.yaml b/docs/examples/substrate-install.yaml index e0c880f261..54e84dc067 100644 --- a/docs/examples/substrate-install.yaml +++ b/docs/examples/substrate-install.yaml @@ -39,10 +39,14 @@ rolloutTimeout: 1m0s # Ingress and egress dataplane: envoy or agentgateway. atenet: dataplane: envoy - # Router timeouts for agent workloads; unset keeps the router defaults (5s, 5m). Envoy only. + # Router settings for agent workloads; unset keeps the router defaults. Envoy only. router: + # Park budget and route timeout; the router defaults are 5s and 5m. parkedRequestBudget: 2m routeTimeout: 30m + # Circuit breaker on Envoy's ext_proc cluster; the router derives twice the + # parking lot (at least 1024) when unset. + extprocMaxRequests: 4096 # Prebuilt images. Both halves are required together: a tag with no repository # would be dropped, leaving a build from source that looks like the release the diff --git a/docs/operator-install.md b/docs/operator-install.md index 51443c6ef6..4bc2153677 100644 --- a/docs/operator-install.md +++ b/docs/operator-install.md @@ -294,6 +294,7 @@ demos, the [benchmarks](../benchmarking/README.md#settings), and the | `atenet.dataplane` | `ATE_ATENET_DATAPLANE` | `--atenet-dataplane` | `envoy` | (global) | | `atenet.egress.additionalExtprocService` | `ATE_ADDITIONAL_EGRESS_EXTPROC_SERVICE` | `--experimental-additional-egress-extproc-service` | — | (global) | | `atenet.egress.credentialProvider` | `ATE_CREDENTIAL_PROVIDER` | `--credential-provider` | — | (global) | +| `atenet.router.extprocMaxRequests` | `ATE_ATENET_EXTPROC_MAX_REQUESTS` | `--atenet-extproc-max-requests` | — | (global) | | `atenet.router.parkedRequestBudget` | `ATE_ATENET_PARKED_REQUEST_BUDGET` | `--atenet-parked-request-budget` | — | (global) | | `atenet.router.routeTimeout` | `ATE_ATENET_ROUTE_TIMEOUT` | `--atenet-route-timeout` | — | (global) | | `clusterSize` | `ATE_INSTALL_CLUSTER_SIZE` | `--cluster-size` | `size0` | (global) | diff --git a/docs/request-parking.md b/docs/request-parking.md index 9f48bff13e..3aee30f7cd 100644 --- a/docs/request-parking.md +++ b/docs/request-parking.md @@ -160,6 +160,13 @@ actor serves. Both are install settings, `--atenet-parked-request-budget` and `terminationGracePeriodSeconds` with the budget so parked requests still finish during a rollout. +A burst of requests larger than the ext_proc circuit breaker is rejected by +Envoy itself, with `503` and `reset reason: overflow`, before the router sees +it. The router derives the breaker as twice `--parked-request-max` (at least +`1024`); the install setting `--atenet-extproc-max-requests` +(`ATE_ATENET_EXTPROC_MAX_REQUESTS`) raises it for hosts dense enough to +synchronize more requests than that. + ## Observability **Metrics** (OpenTelemetry, meter `atenet-router`): diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index ad0efd9d51..0e2965614e 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -244,8 +244,9 @@ spec: # default of 5m, which suits an agent relaying a model completion: the # request stays open for the whole generation. Set --route-timeout # lower to cap how long one turn may hold a request open. - # ate-setup appends --route-timeout and --parked-request-budget here from - # its atenet.router.* settings and grows the grace period above with the budget. + # ate-setup appends --route-timeout, --parked-request-budget and + # --extproc-max-requests here from its atenet.router.* settings and grows + # the grace period above with the budget. # # A turn still running after the drain route budget (10s) does not # survive a shutdown, and raising --route-timeout does not change that: