e2e: add multi-node rollout test on EKS

Makefile

@@ -117,7 +117,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex>
 		E2E_REGISTRY_USER=$(E2E_REGISTRY_USER) E2E_REGISTRY_PASSWORD=$(E2E_REGISTRY_PASSWORD) \
 		$(if $(RELEASED_OPERATOR_TAG),E2E_OPERATOR_RELEASE_TAG=$(RELEASED_OPERATOR_TAG)) \
 		$(if $(RELEASED_OPERATOR_IMG),E2E_OPERATOR_RELEASED_IMG=$(IMG_BINK_RELEASED)) \
-		go test -timeout 60m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') .
+		go test -timeout 90m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') .
 
 # EKS e2e settings
 EKS_CLUSTER_NAME ?=
@@ -146,7 +146,7 @@ e2e-eks: ## Run e2e tests against EKS. V=1 for verbose. RUN=<regex> to filter.
 		E2E_REGISTRY_USER="$(EKS_REGISTRY_USER)" \
 		E2E_REGISTRY_PASSWORD="$(EKS_REGISTRY_PASSWORD)" \
 		ARTIFACTS="$(ARTIFACTS)" \
-		go test -timeout 30m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') .
+		go test -timeout 60m -count=1 -parallel 2 $(if $(V),-v) $(if $(RUN),-run '$(RUN)') .
 
 ##@ Build
 

test/e2e/bootcnode_test.go

@@ -21,6 +21,7 @@ import (
 	meta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1"
@@ -1294,3 +1295,184 @@ func TestOperatorRestartResilience(t *testing.T) {
 		Should(poolAllUpdated(1, env.NodeImageUpdateDigest()))
 	t.Logf("Operator functioned normally after restart: rollout completed")
 }
+
+// TestMultiNodeRollout provisions a pool of multiple worker nodes
+// (COS-5185: scale testing for bootc-operator) and verifies a rollout
+// updates every node in the pool, respecting maxUnavailable, and that
+// the pool status correctly reflects all nodes once the rollout
+// completes. Unlike TestMidRolloutImageChange (bink-only, 2 nodes,
+// exercises mid-rollout image switches), this test targets a larger
+// pool and runs on EKS to validate behavior on real cloud nodes rather
+// than the simulated bink provider, since upstream CI's GitHub runners
+// cannot afford enough memory/CPU for multi-node scale testing.
+func TestMultiNodeRollout(t *testing.T) {
+	e2eutil.Providers(t, "bink", "eks")
+	// Serial on bink: this pool's three nodes plus any concurrently
+	// running test's nodes contend for the GitHub runner's memory. On
+	// EKS the nodes are real cloud VMs, so the test stays parallel
+	// there to keep the suite's wall time down.
+	if e2eutil.ProviderName() != "bink" {
+		t.Parallel()
+	}
+	g := NewWithT(t)
+	g.SetDefaultEventuallyTimeout(pollTimeout)
+	g.SetDefaultEventuallyPollingInterval(pollInterval)
+
+	env := e2eutil.New(t)
+
+	const nodeCount = 3
+	nodes := make([]string, nodeCount)
+	for i := range nodes {
+		nodes[i] = env.AddNode(t)
+	}
+
+	ctx := context.Background()
+
+	// Phase 1: Create pool with original image and wait for all nodes Idle.
+	// maxUnavailable bounds how many nodes may be disrupted at once.
+	const maxUnavailable = 1
+	pool := env.NewPool("scale", env.NodeImageDigestedPullSpec(),
+		testutil.WithMaxUnavailable(intstr.FromInt(maxUnavailable)))
+	g.Expect(env.Client.Create(ctx, pool)).To(Succeed())
+
+	// Verify the daemon pod actually reaches Running on every node before
+	// waiting for Idle - a pod stuck in e.g. ImagePullBackOff would
+	// otherwise just show up as a confusing Idle-wait timeout below.
+	for _, nodeName := range nodes {
+		g.Eventually(func() ([]corev1.Pod, error) {
+			var pods corev1.PodList
+			err := env.Client.List(ctx, &pods,
+				client.InNamespace(testutil.OperatorNamespaceName),
+				client.MatchingFields{"spec.nodeName": nodeName},
+				client.MatchingLabels{
+					"app.kubernetes.io/name":      "bootc-operator",
+					"app.kubernetes.io/component": "daemon",
+				},
+			)
+			return pods.Items, err
+		}).WithTimeout(3*time.Minute).Should(ConsistOf(And(
+			HaveField("Spec.NodeName", nodeName),
+			HaveField("Status.Phase", corev1.PodRunning),
+		)), "expected exactly one running daemon pod on %s", nodeName)
+	}
+
+	for _, nodeName := range nodes {
+		testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute)
+	}
+
+	t.Logf("All %d nodes are Idle with original image", nodeCount)
+
+	g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).
+		Should(poolAllUpdated(nodeCount, env.NodeImageDigest()))
+
+	// Phase 2: Patch pool to the update image.
+	updateRef := env.NodeImageUpdateDigestedPullSpec()
+
+	modified := pool.DeepCopy()
+	modified.Spec.Image.Ref = updateRef
+	g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed())
+	*pool = *modified
+
+	t.Logf("Patched pool to update image %s", updateRef)
+
+	// Phase 3: Wait for every node to be Idle on the update digest - proves
+	// the rollout reached the full pool, not just a subset - while
+	// continuously enforcing maxUnavailable. Both checks share one polling
+	// loop so the invariant is sampled throughout the rollout rather than
+	// once before it starts, and so the wait ends as soon as the rollout
+	// does. Asserting the invariant once up front would pass trivially,
+	// since nothing is disrupted yet at that point.
+	//
+	// A node that stays Degraded aborts the wait early and reports the
+	// daemon's own error message. The daemon clears Degraded on the next
+	// successful reconcile, so a retried transient error must not fail the
+	// test; only a node degraded for longer than degradedGrace does.
+	const degradedGrace = 90 * time.Second
+	degradedSince := make(map[string]time.Time)
+
+	g.Eventually(func(g Gomega) {
+		// Snapshot the whole pool in a single List. The assertions below
+		// abort the iteration at the first node that is behind, so the
+		// invariant has to be evaluated across every node before any of
+		// them run. It must also be one List rather than a Get per node:
+		// a List is a consistent snapshot at one resourceVersion, whereas
+		// sequential Gets can straddle a slot handoff and assemble a view
+		// in which two nodes hold a slot, a state that never existed.
+		var bnList bootcv1alpha1.BootcNodeList
+		g.Expect(env.Client.List(ctx, &bnList)).To(Succeed())
+
+		inPool := make(map[string]bool, len(nodes))
+		for _, nodeName := range nodes {
+			inPool[nodeName] = true
+		}
+		bns := make([]bootcv1alpha1.BootcNode, 0, len(nodes))
+		for i := range bnList.Items {
+			if inPool[bnList.Items[i].Name] {
+				bns = append(bns, bnList.Items[i])
+			}
+		}
+		g.Expect(bns).To(HaveLen(len(nodes)), "expected a BootcNode per pool node")
+
+		// maxUnavailable bounds reboot slots, not staging. Staging only
+		// downloads the image and disrupts nothing, so the controller lets
+		// the whole pool stage at once and gates cordon/drain/reboot. A
+		// node holds a slot exactly while it carries the in-reboot-slot
+		// annotation, which is also how the controller counts them.
+		//
+		// Exceeding the bound is a violated invariant rather than a state
+		// to wait out - retrying would let a later poll mask it.
+		inRebootSlot := 0
+		for i := range bns {
+			if metav1.HasAnnotation(bns[i].ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) {
+				inRebootSlot++
+			}
+		}
+		if inRebootSlot > maxUnavailable {
+			StopTrying(fmt.Sprintf(
+				"maxUnavailable=%d violated: %d nodes hold a reboot slot at once",
+				maxUnavailable, inRebootSlot,
+			)).Now()
+		}
+
+		for i := range bns {
+			bn := &bns[i]
+			if cond := meta.FindStatusCondition(
+				bn.Status.Conditions, bootcv1alpha1.NodeDegraded,
+			); cond != nil && cond.Status == metav1.ConditionTrue {
+				first, seen := degradedSince[bn.Name]
+				switch {
+				case !seen:
+					degradedSince[bn.Name] = time.Now()
+				case time.Since(first) > degradedGrace:
+					StopTrying(fmt.Sprintf(
+						"node %s degraded for over %s during rollout: %s: %s",
+						bn.Name, degradedGrace, cond.Reason, cond.Message,
+					)).Now()
+				}
+			} else {
+				delete(degradedSince, bn.Name)
+			}
+		}
+
+		for i := range bns {
+			bn := &bns[i]
+			g.Expect(bn.Status.Conditions).To(ContainElement(And(
+				HaveField("Type", bootcv1alpha1.NodeIdle),
+				HaveField("Status", metav1.ConditionTrue),
+				HaveField("Reason", bootcv1alpha1.NodeReasonIdle),
+			)), "node %s is not Idle yet", bn.Name)
+
+			g.Expect(bn.Status.Booted).NotTo(BeNil())
+			g.Expect(bn.Status.Booted).To(
+				imageMatchesDigest(env.NodeImageUpdateDigest()),
+				"node %s has not booted the update digest yet", bn.Name,
+			)
+		}
+	}).WithTimeout(20 * time.Minute).Should(Succeed())
+
+	t.Logf("All %d nodes are Idle with update image", nodeCount)
+
+	// Phase 4: Verify final pool status reflects all nodes updated.
+	g.Eventually(fetchPoolStatus(ctx, env.Client, pool)).
+		Should(poolAllUpdated(nodeCount, env.NodeImageUpdateDigest()))
+}

test/e2e/e2eutil/env.go

@@ -108,10 +108,7 @@ func New(t *testing.T) *Env {
 
 	k8sClient := buildClient(t, kubeconfigPath)
 
-	providerName := os.Getenv("E2E_PROVIDER")
-	if providerName == "" {
-		providerName = "bink"
-	}
+	providerName := ProviderName()
 
 	var (
 		provider     NodeProvider

test/e2e/e2eutil/provider.go

@@ -15,15 +15,20 @@ type NodeProvider interface {
 	RemoveNode(ctx context.Context, nodeName string) error
 }
 
+// ProviderName returns the active E2E_PROVIDER, defaulting to bink.
+func ProviderName() string {
+	if p := os.Getenv("E2E_PROVIDER"); p != "" {
+		return p
+	}
+	return "bink"
+}
+
 // Providers skips the test if the current E2E_PROVIDER is not in the
 // supported list. Call at the top of each test function.
 func Providers(t *testing.T, supported ...string) {
 	t.Helper()
 
-	provider := os.Getenv("E2E_PROVIDER")
-	if provider == "" {
-		provider = "bink"
-	}
+	provider := ProviderName()
 	for _, s := range supported {
 		if s == provider {
 			return