From 5085bf512bd86cedb720a83a14620fdaa1708bb9 Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Fri, 16 Oct 2026 17:33:09 +0530 Subject: [PATCH 1/2] e2e: add eks test case for scaling node Signed-off-by: Prachiti Talgulkar --- test/e2e/bootcnode_test.go | 176 +++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index aa6fe40..9f18e3b 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -21,6 +21,7 @@ import ( meta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" @@ -1294,3 +1295,178 @@ func TestOperatorRestartResilience(t *testing.T) { Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) t.Logf("Operator functioned normally after restart: rollout completed") } + +// TestMultiNodeRollout provisions a pool of multiple worker nodes +// (COS-5185: scale testing for bootc-operator) and verifies a rollout +// updates every node in the pool, respecting maxUnavailable, and that +// the pool status correctly reflects all nodes once the rollout +// completes. Unlike TestMidRolloutImageChange (bink-only, 2 nodes, +// exercises mid-rollout image switches), this test targets a larger +// pool and runs on EKS to validate behavior on real cloud nodes rather +// than the simulated bink provider, since upstream CI's GitHub runners +// cannot afford enough memory/CPU for multi-node scale testing. +func TestMultiNodeRollout(t *testing.T) { + e2eutil.Providers(t, "bink", "eks") + t.Parallel() + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + + const nodeCount = 3 + nodes := make([]string, nodeCount) + for i := range nodes { + nodes[i] = env.AddNode(t) + } + + ctx := context.Background() + + // Phase 1: Create pool with original image and wait for all nodes Idle. + // maxUnavailable bounds how many nodes may be disrupted at once. + const maxUnavailable = 1 + pool := env.NewPool("scale", env.NodeImageDigestedPullSpec(), + testutil.WithMaxUnavailable(intstr.FromInt(maxUnavailable))) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + // Verify the daemon pod actually reaches Running on every node before + // waiting for Idle - a pod stuck in e.g. ImagePullBackOff would + // otherwise just show up as a confusing Idle-wait timeout below. + for _, nodeName := range nodes { + g.Eventually(func() ([]corev1.Pod, error) { + var pods corev1.PodList + err := env.Client.List(ctx, &pods, + client.InNamespace(testutil.OperatorNamespaceName), + client.MatchingFields{"spec.nodeName": nodeName}, + client.MatchingLabels{ + "app.kubernetes.io/name": "bootc-operator", + "app.kubernetes.io/component": "daemon", + }, + ) + return pods.Items, err + }).WithTimeout(3*time.Minute).Should(ConsistOf(And( + HaveField("Spec.NodeName", nodeName), + HaveField("Status.Phase", corev1.PodRunning), + )), "expected exactly one running daemon pod on %s", nodeName) + } + + for _, nodeName := range nodes { + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute) + } + + t.Logf("All %d nodes are Idle with original image", nodeCount) + + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(nodeCount, env.NodeImageDigest())) + + // Phase 2: Patch pool to the update image. + updateRef := env.NodeImageUpdateDigestedPullSpec() + + modified := pool.DeepCopy() + modified.Spec.Image.Ref = updateRef + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) + *pool = *modified + + t.Logf("Patched pool to update image %s", updateRef) + + // Phase 3: Wait for every node to be Idle on the update digest - proves + // the rollout reached the full pool, not just a subset - while + // continuously enforcing maxUnavailable. Both checks share one polling + // loop so the invariant is sampled throughout the rollout rather than + // once before it starts, and so the wait ends as soon as the rollout + // does. Asserting the invariant once up front would pass trivially, + // since nothing is disrupted yet at that point. + // + // A node that stays Degraded aborts the wait early and reports the + // daemon's own error message. The daemon clears Degraded on the next + // successful reconcile, so a retried transient error must not fail the + // test; only a node degraded for longer than degradedGrace does. + const degradedGrace = 90 * time.Second + degradedSince := make(map[string]time.Time) + + g.Eventually(func(g Gomega) { + // Snapshot the whole pool in a single List. The assertions below + // abort the iteration at the first node that is behind, so the + // invariant has to be evaluated across every node before any of + // them run. It must also be one List rather than a Get per node: + // a List is a consistent snapshot at one resourceVersion, whereas + // sequential Gets can straddle a slot handoff and assemble a view + // in which two nodes hold a slot, a state that never existed. + var bnList bootcv1alpha1.BootcNodeList + g.Expect(env.Client.List(ctx, &bnList)).To(Succeed()) + + inPool := make(map[string]bool, len(nodes)) + for _, nodeName := range nodes { + inPool[nodeName] = true + } + bns := make([]bootcv1alpha1.BootcNode, 0, len(nodes)) + for i := range bnList.Items { + if inPool[bnList.Items[i].Name] { + bns = append(bns, bnList.Items[i]) + } + } + g.Expect(bns).To(HaveLen(len(nodes)), "expected a BootcNode per pool node") + + // maxUnavailable bounds reboot slots, not staging. Staging only + // downloads the image and disrupts nothing, so the controller lets + // the whole pool stage at once and gates cordon/drain/reboot. A + // node holds a slot exactly while it carries the in-reboot-slot + // annotation, which is also how the controller counts them. + // + // Exceeding the bound is a violated invariant rather than a state + // to wait out - retrying would let a later poll mask it. + inRebootSlot := 0 + for i := range bns { + if metav1.HasAnnotation(bns[i].ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) { + inRebootSlot++ + } + } + if inRebootSlot > maxUnavailable { + StopTrying(fmt.Sprintf( + "maxUnavailable=%d violated: %d nodes hold a reboot slot at once", + maxUnavailable, inRebootSlot, + )).Now() + } + + for i := range bns { + bn := &bns[i] + if cond := meta.FindStatusCondition( + bn.Status.Conditions, bootcv1alpha1.NodeDegraded, + ); cond != nil && cond.Status == metav1.ConditionTrue { + first, seen := degradedSince[bn.Name] + switch { + case !seen: + degradedSince[bn.Name] = time.Now() + case time.Since(first) > degradedGrace: + StopTrying(fmt.Sprintf( + "node %s degraded for over %s during rollout: %s: %s", + bn.Name, degradedGrace, cond.Reason, cond.Message, + )).Now() + } + } else { + delete(degradedSince, bn.Name) + } + } + + for i := range bns { + bn := &bns[i] + g.Expect(bn.Status.Conditions).To(ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.NodeReasonIdle), + )), "node %s is not Idle yet", bn.Name) + + g.Expect(bn.Status.Booted).NotTo(BeNil()) + g.Expect(bn.Status.Booted).To( + imageMatchesDigest(env.NodeImageUpdateDigest()), + "node %s has not booted the update digest yet", bn.Name, + ) + } + }).WithTimeout(20 * time.Minute).Should(Succeed()) + + t.Logf("All %d nodes are Idle with update image", nodeCount) + + // Phase 4: Verify final pool status reflects all nodes updated. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(nodeCount, env.NodeImageUpdateDigest())) +} From 75b6808210ddb34fa6041eb2400974cfb718c18f Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Wed, 21 Oct 2026 18:14:51 +0530 Subject: [PATCH 2/2] ci: raise e2e timeouts for the multi-node rollout test TestMultiNodeRollout takes ~24 min on the bink runners, pushing the full suite past the previous 60m budget (k8s 1.35 died with panic: test timed out after 1h0m0s while two tests were still in flight). On EKS the suite now provisions five EC2 nodes across three tests, which does not reliably fit the old 30m either. e2e: 60m -> 90m, e2e-eks: 30m -> 60m. Co-Authored-By: Claude Code Signed-off-by: Prachiti Talgulkar --- Makefile | 4 ++-- test/e2e/bootcnode_test.go | 8 +++++++- test/e2e/e2eutil/env.go | 5 +---- test/e2e/e2eutil/provider.go | 13 +++++++++---- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 09bf763..88b770b 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN= E2E_REGISTRY_USER=$(E2E_REGISTRY_USER) E2E_REGISTRY_PASSWORD=$(E2E_REGISTRY_PASSWORD) \ $(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \ $(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \ - go test -timeout 60m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . + go test -timeout 90m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . # EKS e2e settings EKS_CLUSTER_NAME ?= @@ -146,7 +146,7 @@ e2e-eks: ## Run e2e tests against EKS. V=1 for verbose. RUN= to filter. E2E_REGISTRY_USER="$(EKS_REGISTRY_USER)" \ E2E_REGISTRY_PASSWORD="$(EKS_REGISTRY_PASSWORD)" \ ARTIFACTS="$(ARTIFACTS)" \ - go test -timeout 30m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . + go test -timeout 60m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') . ##@ Build diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index 9f18e3b..f40180b 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -1307,7 +1307,13 @@ func TestOperatorRestartResilience(t *testing.T) { // cannot afford enough memory/CPU for multi-node scale testing. func TestMultiNodeRollout(t *testing.T) { e2eutil.Providers(t, "bink", "eks") - t.Parallel() + // Serial on bink: this pool's three nodes plus any concurrently + // running test's nodes contend for the GitHub runner's memory. On + // EKS the nodes are real cloud VMs, so the test stays parallel + // there to keep the suite's wall time down. + if e2eutil.ProviderName() != "bink" { + t.Parallel() + } g := NewWithT(t) g.SetDefaultEventuallyTimeout(pollTimeout) g.SetDefaultEventuallyPollingInterval(pollInterval) diff --git a/test/e2e/e2eutil/env.go b/test/e2e/e2eutil/env.go index 815d6ab..e7dfdf0 100644 --- a/test/e2e/e2eutil/env.go +++ b/test/e2e/e2eutil/env.go @@ -108,10 +108,7 @@ func New(t *testing.T) *Env { k8sClient := buildClient(t, kubeconfigPath) - providerName := os.Getenv("E2E_PROVIDER") - if providerName == "" { - providerName = "bink" - } + providerName := ProviderName() var ( provider NodeProvider diff --git a/test/e2e/e2eutil/provider.go b/test/e2e/e2eutil/provider.go index 3154c99..fedb044 100644 --- a/test/e2e/e2eutil/provider.go +++ b/test/e2e/e2eutil/provider.go @@ -15,15 +15,20 @@ type NodeProvider interface { RemoveNode(ctx context.Context, nodeName string) error } +// ProviderName returns the active E2E_PROVIDER, defaulting to bink. +func ProviderName() string { + if p := os.Getenv("E2E_PROVIDER"); p != "" { + return p + } + return "bink" +} + // Providers skips the test if the current E2E_PROVIDER is not in the // supported list. Call at the top of each test function. func Providers(t *testing.T, supported ...string) { t.Helper() - provider := os.Getenv("E2E_PROVIDER") - if provider == "" { - provider = "bink" - } + provider := ProviderName() for _, s := range supported { if s == provider { return