From f8aabf0073df3fea48ee7487bb3337e92dee04e3 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Thu, 22 Oct 2026 14:08:56 +0000 Subject: [PATCH 1/2] e2e: add control-plane role support to NodeProvider Extend the NodeProvider.AddNode interface with a role parameter so tests can provision control-plane nodes in addition to workers. The bink provider passes --role to the bink CLI when set. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/e2eutil/env.go | 13 +++++++++++-- test/e2e/e2eutil/provider.go | 2 +- test/e2e/e2eutil/provider_bink.go | 4 ++++ test/e2e/e2eutil/provider_eks.go | 1 + test/e2e/e2eutil/provider_eks_test.go | 2 +- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/test/e2e/e2eutil/env.go b/test/e2e/e2eutil/env.go index 815d6abf..1f6ff3c0 100644 --- a/test/e2e/e2eutil/env.go +++ b/test/e2e/e2eutil/env.go @@ -195,6 +195,7 @@ type NodeOption func(*nodeConfig) type nodeConfig struct { labels map[string]string + role string } // WithLabel adds a label to the provisioned node. This is in addition @@ -208,6 +209,14 @@ func WithLabel(key, value string) NodeOption { } } +// WithRole sets the node role (e.g. "control-plane"). Defaults to +// "worker" when empty. +func WithRole(role string) NodeOption { + return func(c *nodeConfig) { + c.role = role + } +} + // AddNode provisions a worker node via the configured provider, waits // for it to be Ready, and returns the node name. The node is labeled // with LabelE2ETest (and any extra labels from WithLabel). @@ -225,8 +234,8 @@ func (e *Env) AddNode(t *testing.T, opts ...NodeOption) string { } ctx := context.Background() - t.Logf("Adding node...") - nodeName, err := e.provider.AddNode(ctx, labels) + t.Logf("Adding node (role=%q)...", cfg.role) + nodeName, err := e.provider.AddNode(ctx, labels, cfg.role) if err != nil { t.Fatalf("adding node: %v", err) } diff --git a/test/e2e/e2eutil/provider.go b/test/e2e/e2eutil/provider.go index 3154c992..3b694d26 100644 --- a/test/e2e/e2eutil/provider.go +++ b/test/e2e/e2eutil/provider.go @@ -11,7 +11,7 @@ import ( // NodeProvider abstracts node provisioning so the same tests can run // against different infrastructure (bink, EKS, etc.). type NodeProvider interface { - AddNode(ctx context.Context, labels map[string]string) (nodeName string, err error) + AddNode(ctx context.Context, labels map[string]string, role string) (nodeName string, err error) RemoveNode(ctx context.Context, nodeName string) error } diff --git a/test/e2e/e2eutil/provider_bink.go b/test/e2e/e2eutil/provider_bink.go index 366f1cc5..8bf2beec 100644 --- a/test/e2e/e2eutil/provider_bink.go +++ b/test/e2e/e2eutil/provider_bink.go @@ -39,6 +39,7 @@ func newBinkProvider( func (p *binkProvider) AddNode( ctx context.Context, labels map[string]string, + role string, ) (string, error) { prefix := labels[LabelE2ETest] if prefix == "" { @@ -55,6 +56,9 @@ func (p *binkProvider) AddNode( "node", "add", nodeName, "--cluster-name", p.clusterName, } + if role != "" { + args = append(args, "--role", role) + } for k, v := range labels { args = append(args, "--label", k+"="+v) } diff --git a/test/e2e/e2eutil/provider_eks.go b/test/e2e/e2eutil/provider_eks.go index 43a90a17..f1708d61 100644 --- a/test/e2e/e2eutil/provider_eks.go +++ b/test/e2e/e2eutil/provider_eks.go @@ -113,6 +113,7 @@ func newEKSProvider( func (p *eksProvider) AddNode( ctx context.Context, labels map[string]string, + _ string, ) (string, error) { out, err := p.ec2Runner.RunInstances(ctx, &ec2.RunInstancesInput{ LaunchTemplate: &ec2types.LaunchTemplateSpecification{ diff --git a/test/e2e/e2eutil/provider_eks_test.go b/test/e2e/e2eutil/provider_eks_test.go index 44f69033..4bcbd290 100644 --- a/test/e2e/e2eutil/provider_eks_test.go +++ b/test/e2e/e2eutil/provider_eks_test.go @@ -81,7 +81,7 @@ func TestEKSAddNode(t *testing.T) { } labels := map[string]string{LabelE2ETest: "testid"} - got, err := provider.AddNode(context.Background(), labels) + got, err := provider.AddNode(context.Background(), labels, "") if err != nil { t.Fatalf("AddNode: %v", err) } From 795c1c13440e676217090c569f8ecd59d5b58ca6 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Thu, 22 Oct 2026 14:09:03 +0000 Subject: [PATCH 2/2] e2e: add control-plane upgrade test Add TestControlPlaneUpgrade to verify the operator can survive upgrading the node it runs on. The test adds a second control-plane node, creates a pool targeting all control-plane nodes, and triggers a rollout. The operator must migrate and recover when its own node reboots. Assisted-by: AI Signed-off-by: Alice Frosi --- test/e2e/controlplane_test.go | 110 ++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 test/e2e/controlplane_test.go diff --git a/test/e2e/controlplane_test.go b/test/e2e/controlplane_test.go new file mode 100644 index 00000000..964f6bf4 --- /dev/null +++ b/test/e2e/controlplane_test.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + "github.com/bootc-dev/bootc-operator/test/e2e/e2eutil" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +// TestControlPlaneUpgrade adds a second control-plane node, creates a pool +// selecting all control-plane nodes, and triggers an OS upgrade. Because the +// operator itself runs on a control-plane node, the rollout eventually +// upgrades the node hosting the operator, forcing it to migrate and recover. +// This validates that the controller can survive upgrading its own node. +func TestControlPlaneUpgrade(t *testing.T) { + e2eutil.Providers(t, "bink") + // Serial: modifies the original control-plane node and cluster + // infrastructure; no other tests should run concurrently. + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + ctx := context.Background() + + // Phase 1: Add a second control-plane node so infrastructure pods can + // migrate when the first one reboots. + cpNode := env.AddNode(t, e2eutil.WithRole("control-plane")) + t.Logf("Added control-plane node %q", cpNode) + + // Phase 2: Create a pool selecting all control-plane nodes. The + // original "controller" node and our new node both carry the + // node-role.kubernetes.io/control-plane label from kubeadm. + pool := env.NewPool("cp-upgrade", + env.NodeImageDigestedPullSpec(), + testutil.WithNodeSelector(testutil.ControlPlaneLabels()), + ) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + t.Logf("Created pool %q selecting control-plane nodes", pool.Name) + + // Wait for both control-plane nodes to reach Idle with the base image. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + WithTimeout(5 * time.Minute). + Should(And( + HaveField("NodeCount", BeNumerically(">=", 2)), + HaveField("UpdatingCount", BeEquivalentTo(0)), + HaveField("Conditions", ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolUpToDate), + HaveField("Status", metav1.ConditionTrue), + ))), + )) + t.Logf("Pool is up to date with base image, %d nodes managed", 2) + + // Phase 3: Trigger the upgrade by patching the pool to the update image. + updateRef := env.NodeImageUpdateDigestedPullSpec() + modified := pool.DeepCopy() + modified.Spec.Image.Ref = updateRef + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) + *pool = *modified + t.Logf("Patched pool to update image %s", updateRef) + + // Phase 4: Wait for the rollout to complete. This involves two reboots + // (one per control-plane node, serialized by maxUnavailable=1). When the + // operator's own node reboots, the operator pod migrates to the other + // control-plane and must resume the rollout. Use a generous timeout: + // each reboot cycle (drain + reboot + rejoin) takes several minutes, and + // the operator needs time to recover after migration. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + WithTimeout(15*time.Minute).WithPolling(10*time.Second). + Should(poolAllUpdated(2, env.NodeImageUpdateDigest()), + "expected all control-plane nodes to complete the upgrade") + + t.Logf("All control-plane nodes upgraded successfully") + + // Phase 5: Verify each managed BootcNode is Idle with the update digest. + var poolObj bootcv1alpha1.BootcNodePool + g.Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(pool), &poolObj)).To(Succeed()) + + var bnList bootcv1alpha1.BootcNodeList + g.Expect(env.Client.List(ctx, &bnList)).To(Succeed()) + for _, bn := range bnList.Items { + testutil.WaitForNodeIdle(t, g, ctx, env.Client, bn.Name, 3*time.Minute, + HaveField("Booted", imageMatchesDigest(env.NodeImageUpdateDigest())), + ) + } + + // Phase 6: Roll back to the original image to leave the cluster clean. + originalRef := env.NodeImageDigestedPullSpec() + rollback := pool.DeepCopy() + rollback.Spec.Image.Ref = originalRef + g.Expect(env.Client.Patch(ctx, rollback, client.MergeFrom(pool))).To(Succeed()) + *pool = *rollback + t.Logf("Rolling back to original image %s", originalRef) + + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + WithTimeout(15*time.Minute).WithPolling(10*time.Second). + Should(poolAllUpdated(2, env.NodeImageDigest()), + "expected rollback to complete on all control-plane nodes") + + t.Logf("Rollback completed, cluster is back on the original image") +}