diff --git a/README.md b/README.md index 53f47e7..516a47d 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,13 @@ makes all of that assertable. - **Known defects that expire.** `KnownDefect` pins a live bug: the test passes while the bug is present and fails the day it is fixed, telling you to replace the pin with a real assertion. It can't quietly outlive the bug. -- **A data plane envtest doesn't have.** `DataPlaneSim` makes Pods Running and - Ready, gives Deployments and StatefulSets status matching their replicas, - and binds PVCs, so controllers that wait on readiness make progress. +- **A data plane envtest doesn't have.** `DataPlaneSim` reports Pods the way + kubelet would, native sidecars and readiness gates included, gives + Deployments and StatefulSets status matching their replicas, and binds PVCs, + so controllers that wait on readiness make progress. `C.FaultPods` breaks + chosen pods (unschedulable, a container failing readiness, a container or + init container crashlooping), so a test can show what a controller does when + they don't. - **Failures that come with evidence.** Every `Case` attaches a failure dump: the op log around the failing assertion, interleaved with the reconcile passes that produced it and with other controllers' writes out of scope. A diff --git a/ctrltest/datasim.go b/ctrltest/datasim.go index 64f4f05..d07df92 100644 --- a/ctrltest/datasim.go +++ b/ctrltest/datasim.go @@ -4,13 +4,13 @@ package ctrltest import ( "context" - "fmt" + "sync" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -18,17 +18,32 @@ import ( // Deployment/StatefulSet controllers, and the volume provisioner. // // Modelled: -// - Pods reach Running with a podIP and Ready/ContainersReady/Initialized/ -// PodScheduled conditions true, and per-container Ready statuses. +// - Pods reach Running with a podIP and the conditions and container +// statuses kubelet reports. Sidecars (init containers with restartPolicy +// Always) run alongside the regular containers and count toward +// ContainersReady; other init containers complete first. +// - Ready follows kubelet: ContainersReady and every readiness gate +// condition True. The simulator never writes a gate condition itself, so +// a pod with a gate nobody sets stays not Ready, as it would on a cluster. +// - Pods made unschedulable, not ready or crashlooping by a PodFault +// (SetPodFaults, C.FaultPods). // - Deployments and StatefulSets report status matching spec.replicas. // - PVCs reach Bound. // -// Not modelled: scheduling failure, eviction, container restart, node pressure, -// real volume attachment, or any ordering between them. +// Not modelled: eviction, node pressure, real volume attachment, actual +// restarts, or any ordering between them. Deployment and StatefulSet status is +// synthetic and does not follow pod faults: envtest runs no ReplicaSet or +// StatefulSet controller, so the pods behind them do not exist to fault. type DataPlaneSim struct { c client.Client interval time.Duration + + // podIPSeq is only touched from the sweep goroutine. podIPSeq int + + // mu guards faults, which tests install from their own goroutines. + mu sync.Mutex + faults map[string]*podFaultRule } // NewDataPlaneSim returns a simulator that patches status on every object it @@ -59,33 +74,18 @@ func (s *DataPlaneSim) tickPods(ctx context.Context) { if err := s.c.List(ctx, list); err != nil { return } + now := metav1.Now() for i := range list.Items { p := &list.Items[i] if !p.DeletionTimestamp.IsZero() { continue } - if podSimSatisfied(p) { + want := s.podStatus(p, s.podFault(p), now) + if equality.Semantic.DeepEqual(want, p.Status) { continue } base := p.DeepCopy() - p.Status.Phase = corev1.PodRunning - if p.Status.PodIP == "" { - s.podIPSeq++ - p.Status.PodIP = fmt.Sprintf("10.244.%d.%d", s.podIPSeq/250, s.podIPSeq%250+1) - p.Status.PodIPs = []corev1.PodIP{{IP: p.Status.PodIP}} - p.Status.HostIP = "10.0.0.1" - } - p.Status.StartTime = &metav1.Time{Time: time.Now()} - for _, ct := range []corev1.PodConditionType{ - corev1.PodScheduled, - corev1.PodInitialized, - corev1.ContainersReady, - corev1.PodReady, - } { - setPodCondition(p, ct) - } - p.Status.ContainerStatuses = containerStatuses(p.Spec.Containers) - p.Status.InitContainerStatuses = initContainerStatuses(p.Spec.InitContainers) + p.Status = want // Strategic merge, not JSON merge: a JSON merge patch replaces the // whole conditions array with this tick's snapshot, deleting any // condition another writer (a controller's readiness gate) added @@ -96,89 +96,6 @@ func (s *DataPlaneSim) tickPods(ctx context.Context) { } } -func podSimSatisfied(p *corev1.Pod) bool { - if p.Status.Phase != corev1.PodRunning || p.Status.PodIP == "" { - return false - } - if len(p.Status.ContainerStatuses) != len(p.Spec.Containers) { - return false - } - want := map[corev1.PodConditionType]bool{ - corev1.PodScheduled: false, - corev1.PodInitialized: false, - corev1.ContainersReady: false, - corev1.PodReady: false, - } - for _, c := range p.Status.Conditions { - if _, ok := want[c.Type]; ok && c.Status == corev1.ConditionTrue { - want[c.Type] = true - } - } - for _, ok := range want { - if !ok { - return false - } - } - return true -} - -func setPodCondition(p *corev1.Pod, t corev1.PodConditionType) { - for i := range p.Status.Conditions { - if p.Status.Conditions[i].Type == t { - if p.Status.Conditions[i].Status != corev1.ConditionTrue { - p.Status.Conditions[i].Status = corev1.ConditionTrue - p.Status.Conditions[i].LastTransitionTime = metav1.Now() - } - return - } - } - p.Status.Conditions = append(p.Status.Conditions, corev1.PodCondition{ - Type: t, - Status: corev1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }) -} - -func containerStatuses(cs []corev1.Container) []corev1.ContainerStatus { - out := make([]corev1.ContainerStatus, 0, len(cs)) - for _, c := range cs { - out = append(out, corev1.ContainerStatus{ - Name: c.Name, - Image: c.Image, - ImageID: "sha256:" + c.Name, - ContainerID: "containerd://" + c.Name, - Ready: true, - Started: ptr.To(true), - State: corev1.ContainerState{ - Running: &corev1.ContainerStateRunning{StartedAt: metav1.Now()}, - }, - }) - } - return out -} - -func initContainerStatuses(cs []corev1.Container) []corev1.ContainerStatus { - out := make([]corev1.ContainerStatus, 0, len(cs)) - for _, c := range cs { - out = append(out, corev1.ContainerStatus{ - Name: c.Name, - Image: c.Image, - ImageID: "sha256:" + c.Name, - ContainerID: "containerd://" + c.Name, - Ready: true, - Started: ptr.To(true), - State: corev1.ContainerState{ - Terminated: &corev1.ContainerStateTerminated{ - ExitCode: 0, - Reason: "Completed", - FinishedAt: metav1.Now(), - }, - }, - }) - } - return out -} - func (s *DataPlaneSim) tickDeployments(ctx context.Context) { list := &appsv1.DeploymentList{} if err := s.c.List(ctx, list); err != nil { diff --git a/ctrltest/podfault.go b/ctrltest/podfault.go new file mode 100644 index 0000000..42388c3 --- /dev/null +++ b/ctrltest/podfault.go @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: Apache-2.0 + +package ctrltest + +import ( + "fmt" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +// PodFault describes how DataPlaneSim should report a Pod as broken. The zero +// value, like a nil *PodFault, is a healthy pod. +// +// Container names cover init containers as well as regular ones, and what a +// fault means depends on which kind it names, following kubelet: +// +// - A regular container or a sidecar (an init container with restartPolicy +// Always) named in NotReady runs but fails its readiness probe; in +// CrashLoop it sits in CrashLoopBackOff. Either way ContainersReady, and +// so Ready, is False. +// - A regular init container named in NotReady never completes; in +// CrashLoop it sits in Init:CrashLoopBackOff. Either way the pod stays +// Pending with Initialized False, every later init container and every +// regular container waits in PodInitializing, and nothing is Ready. +// - A crashlooping sidecar blocks the init containers after it the same way, +// because kubelet starts the next one only once a sidecar has started. A +// NotReady sidecar has started, so it does not. +// +// Restarts are modelled only as status: RestartCount climbs on kubelet's +// backoff (10s doubling to 5m), and nothing actually restarts. +type PodFault struct { + // Unscheduled reports the pod as the scheduler leaves one it cannot + // place: Pending, PodScheduled False with reason Unschedulable, no podIP + // and no container statuses. Envtest has no binding to undo, so this + // applies even to a pod the simulator already started. + Unscheduled bool + + // NotReady names containers that run but never pass readiness. + NotReady []string + + // CrashLoop names containers stuck in CrashLoopBackOff. + CrashLoop []string +} + +// PodFaultFunc decides, on every sweep, which fault a pod has. It returns nil +// for a healthy pod. +// +// A predicate rather than a list of pod names, because the pods a test wants +// broken are usually defined by what they run, not what they are called: a +// rollout replaces them with new UIDs and often the same names, and "every pod +// on image X" has to keep matching across that. +type PodFaultFunc func(*corev1.Pod) *PodFault + +// podFaultRule is one namespace's installed PodFaultFunc, with where to report +// a fault that names a container the pod does not have. +type podFaultRule struct { + fn PodFaultFunc + report func(format string, args ...any) + + // reported holds "/" for every unknown name already + // reported, so a sweep every quarter second reports each one once. + reported map[string]bool +} + +// SetPodFaults installs fn for every pod in ns, replacing any earlier one. A +// nil fn heals the namespace. report receives one message per pod and +// unknown container name a fault mentions, since such a fault does nothing +// and a test relying on it would otherwise pass for the wrong reason. +// +// Once SetPodFaults returns, the simulator will not call the previous report +// again, which is what lets C.FaultPods hand it a test's Errorf and clear it in +// Cleanup without a late sweep logging into a finished test. +func (s *DataPlaneSim) SetPodFaults( + ns string, + fn PodFaultFunc, + report func(format string, args ...any), +) { + s.mu.Lock() + defer s.mu.Unlock() + if fn == nil { + delete(s.faults, ns) + return + } + if s.faults == nil { + s.faults = map[string]*podFaultRule{} + } + s.faults[ns] = &podFaultRule{fn: fn, report: report, reported: map[string]bool{}} +} + +// podFault evaluates the namespace's rule for p, reporting unknown container +// names. The rule's fn runs outside the lock, so a slow predicate cannot stall +// SetPodFaults, and the report is re-checked under it, so a rule replaced in +// the meantime is never reported through. +func (s *DataPlaneSim) podFault(p *corev1.Pod) *PodFault { + s.mu.Lock() + rule := s.faults[p.Namespace] + s.mu.Unlock() + if rule == nil { + return nil + } + f := rule.fn(p) + if f == nil { + return nil + } + + var unknown []string + for _, name := range slices.Concat(f.NotReady, f.CrashLoop) { + if !podHasContainer(p, name) { + unknown = append(unknown, name) + } + } + if len(unknown) == 0 { + return f + } + s.mu.Lock() + defer s.mu.Unlock() + if s.faults[p.Namespace] != rule || rule.report == nil { + return f + } + for _, name := range unknown { + key := string(p.UID) + "/" + name + if rule.reported[key] { + continue + } + rule.reported[key] = true + rule.report("PodFault for pod %s/%s names container %q, which the pod does not have", + p.Namespace, p.Name, name) + } + return f +} + +func podHasContainer(p *corev1.Pod, name string) bool { + for _, c := range slices.Concat(p.Spec.InitContainers, p.Spec.Containers) { + if c.Name == name { + return true + } + } + return false +} + +// containerHealth is what a fault makes of one container. +type containerHealth int + +const ( + healthy containerHealth = iota + notReady + crashLoop +) + +func (f *PodFault) health(name string) containerHealth { + switch { + case f == nil: + return healthy + case slices.Contains(f.CrashLoop, name): + return crashLoop + case slices.Contains(f.NotReady, name): + return notReady + default: + return healthy + } +} + +// crashLoopBackoff is kubelet's restart backoff: 10s, doubling, capped at 5m. +func crashLoopBackoff(restarts int32) time.Duration { + d := 10 * time.Second + for i := int32(1); i < restarts && d < 5*time.Minute; i++ { + d *= 2 + } + return min(d, 5*time.Minute) +} + +// podStatus computes the status kubelet and the scheduler would report for p +// under f, from p's current status. Timestamps are carried over wherever the +// state they date has not changed, so a pod that already matches comes back +// equal under podStatusShape and the sweep writes nothing. +func (s *DataPlaneSim) podStatus(p *corev1.Pod, f *PodFault, now metav1.Time) corev1.PodStatus { + st := *p.Status.DeepCopy() + + if f != nil && f.Unscheduled { + st.Phase = corev1.PodPending + st.PodIP, st.PodIPs, st.HostIP, st.HostIPs = "", nil, "", nil + st.StartTime = nil + st.ContainerStatuses, st.InitContainerStatuses = nil, nil + setCondition(&st, corev1.PodScheduled, false, "Unschedulable", now) + setCondition(&st, corev1.PodInitialized, false, "", now) + setCondition(&st, corev1.ContainersReady, false, "ContainersNotReady", now) + setCondition(&st, corev1.PodReady, false, "ContainersNotReady", now) + return st + } + + if st.PodIP == "" { + s.podIPSeq++ + st.PodIP = fmt.Sprintf("10.244.%d.%d", s.podIPSeq/250, s.podIPSeq%250+1) + st.PodIPs = []corev1.PodIP{{IP: st.PodIP}} + st.HostIP = "10.0.0.1" + } + if st.StartTime == nil { + st.StartTime = &now + } + + // Init containers start in order, and one that has not finished (or, for a + // sidecar, not started) holds back everything after it. + blocked := false + allReady := true + inits := make([]corev1.ContainerStatus, 0, len(p.Spec.InitContainers)) + for _, c := range p.Spec.InitContainers { + prev := findContainerStatus(p.Status.InitContainerStatuses, c.Name) + sidecar := c.RestartPolicy != nil && *c.RestartPolicy == corev1.ContainerRestartPolicyAlways + var cs corev1.ContainerStatus + switch { + case blocked: + cs = waitingStatus(c, prev, "PodInitializing") + case sidecar: + cs = runningContainerStatus(c, prev, f.health(c.Name), now) + blocked = cs.Started == nil || !*cs.Started + allReady = allReady && cs.Ready + default: + switch f.health(c.Name) { + case healthy: + cs = completedStatus(c, prev, now) + case notReady: + cs = runningContainerStatus(c, prev, notReady, now) + blocked = true + case crashLoop: + cs = runningContainerStatus(c, prev, crashLoop, now) + blocked = true + } + } + inits = append(inits, cs) + } + + mains := make([]corev1.ContainerStatus, 0, len(p.Spec.Containers)) + for _, c := range p.Spec.Containers { + prev := findContainerStatus(p.Status.ContainerStatuses, c.Name) + var cs corev1.ContainerStatus + if blocked { + cs = waitingStatus(c, prev, "PodInitializing") + } else { + cs = runningContainerStatus(c, prev, f.health(c.Name), now) + } + allReady = allReady && cs.Ready + mains = append(mains, cs) + } + st.InitContainerStatuses, st.ContainerStatuses = inits, mains + + st.Phase = corev1.PodRunning + if blocked { + st.Phase = corev1.PodPending + } + containersReady := allReady && !blocked + gatesReady := true + for _, g := range p.Spec.ReadinessGates { + gatesReady = gatesReady && conditionTrue(st.Conditions, g.ConditionType) + } + readyReason := "" + switch { + case !containersReady: + readyReason = "ContainersNotReady" + case !gatesReady: + readyReason = "ReadinessGatesNotReady" + } + + setCondition(&st, corev1.PodScheduled, true, "", now) + setCondition( + &st, + corev1.PodInitialized, + !blocked, + notReason(blocked, "ContainersNotInitialized"), + now, + ) + setCondition(&st, corev1.ContainersReady, containersReady, + notReason(!containersReady, "ContainersNotReady"), now) + setCondition(&st, corev1.PodReady, containersReady && gatesReady, readyReason, now) + return st +} + +func notReason(failing bool, reason string) string { + if failing { + return reason + } + return "" +} + +func conditionTrue(conds []corev1.PodCondition, t corev1.PodConditionType) bool { + for _, c := range conds { + if c.Type == t { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +// setCondition sets t's status and reason, moving LastTransitionTime only when +// the status changes, as kubelet does. +func setCondition( + st *corev1.PodStatus, + t corev1.PodConditionType, + ok bool, + reason string, + now metav1.Time, +) { + status := corev1.ConditionFalse + if ok { + status = corev1.ConditionTrue + } + for i := range st.Conditions { + c := &st.Conditions[i] + if c.Type != t { + continue + } + if c.Status != status { + c.LastTransitionTime = now + } + c.Status, c.Reason = status, reason + return + } + st.Conditions = append(st.Conditions, corev1.PodCondition{ + Type: t, + Status: status, + Reason: reason, + LastTransitionTime: now, + }) +} + +func findContainerStatus(list []corev1.ContainerStatus, name string) *corev1.ContainerStatus { + for i := range list { + if list[i].Name == name { + return &list[i] + } + } + return nil +} + +func baseContainerStatus(c corev1.Container, prev *corev1.ContainerStatus) corev1.ContainerStatus { + cs := corev1.ContainerStatus{ + Name: c.Name, + Image: c.Image, + ImageID: "sha256:" + c.Name, + ContainerID: "containerd://" + c.Name, + } + if prev != nil { + cs.RestartCount = prev.RestartCount + cs.LastTerminationState = prev.LastTerminationState + } + return cs +} + +// runningContainerStatus covers a container kubelet has started: Running and +// Ready when healthy, Running and not Ready when failing readiness, and +// Waiting in CrashLoopBackOff, with RestartCount climbing on kubelet's backoff, +// when crashlooping. +func runningContainerStatus( + c corev1.Container, + prev *corev1.ContainerStatus, + h containerHealth, + now metav1.Time, +) corev1.ContainerStatus { + cs := baseContainerStatus(c, prev) + if h == crashLoop { + cs.Started = ptr.To(false) + cs.State.Waiting = &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"} + var last *corev1.ContainerStateTerminated + if prev != nil { + last = prev.LastTerminationState.Terminated + } + if last == nil || now.Sub(last.FinishedAt.Time) >= crashLoopBackoff(cs.RestartCount) { + cs.RestartCount++ + cs.LastTerminationState = corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 1, + Reason: "Error", + FinishedAt: now, + }, + } + } + return cs + } + cs.Started = ptr.To(true) + cs.Ready = h == healthy + if prev != nil && prev.State.Running != nil { + cs.State.Running = prev.State.Running + } else { + cs.State.Running = &corev1.ContainerStateRunning{StartedAt: now} + } + return cs +} + +func completedStatus( + c corev1.Container, + prev *corev1.ContainerStatus, + now metav1.Time, +) corev1.ContainerStatus { + cs := baseContainerStatus(c, prev) + cs.Ready = true + cs.Started = ptr.To(false) + if prev != nil && prev.State.Terminated != nil && prev.State.Terminated.ExitCode == 0 { + cs.State.Terminated = prev.State.Terminated + } else { + cs.State.Terminated = &corev1.ContainerStateTerminated{ + ExitCode: 0, + Reason: "Completed", + FinishedAt: now, + } + } + return cs +} + +func waitingStatus( + c corev1.Container, + prev *corev1.ContainerStatus, + reason string, +) corev1.ContainerStatus { + cs := baseContainerStatus(c, prev) + cs.Started = ptr.To(false) + cs.State.Waiting = &corev1.ContainerStateWaiting{Reason: reason} + return cs +} + +// FaultPods breaks pods in this case's namespace: on every sweep, the data +// plane simulator reports each pod as fn says, and as healthy where fn +// returns nil. A later call replaces fn; FaultPods(nil) heals the namespace. +// The fault is lifted when the test ends. +// +// A fault naming a container its pod does not have fails the test, once per +// pod and name, because it would otherwise silently do nothing. +func (c *C) FaultPods(fn PodFaultFunc) { + c.Helper() + sim := c.Suite.sim + if sim == nil { + c.Fatalf( + "FaultPods needs the data plane simulator, and Options.SimInterval is not positive", + ) + } + if fn == nil { + sim.SetPodFaults(c.NS, nil, nil) + return + } + sim.SetPodFaults(c.NS, fn, c.Errorf) + c.Cleanup(func() { sim.SetPodFaults(c.NS, nil, nil) }) +} diff --git a/ctrltest/podfault_test.go b/ctrltest/podfault_test.go new file mode 100644 index 0000000..3ec4333 --- /dev/null +++ b/ctrltest/podfault_test.go @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: Apache-2.0 + +package ctrltest + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "github.com/multigres/testkit/assert" +) + +const gateCondition corev1.PodConditionType = "example.com/gate" + +// faultPod is a pod with every container kind the simulator distinguishes: a +// regular init container ("setup"), a sidecar ("db", the shape of a Postgres +// run as a native sidecar), a regular init container after it ("migrate"), +// and a regular container ("app"). +func faultPod(name string, gates ...corev1.PodConditionType) *corev1.Pod { + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", UID: types.UID("uid-" + name)}, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Name: "setup", Image: "i"}, + { + Name: "db", + Image: "i", + RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + }, + {Name: "migrate", Image: "i"}, + }, + Containers: []corev1.Container{{Name: "app", Image: "i"}}, + }, + } + for _, g := range gates { + p.Spec.ReadinessGates = append( + p.Spec.ReadinessGates, + corev1.PodReadinessGate{ConditionType: g}, + ) + } + return p +} + +type simFixture struct { + c *assert.C + sim *DataPlaneSim + base client.Client + patches atomic.Int32 +} + +func newSimFixture(t *testing.T, pods ...*corev1.Pod) *simFixture { + t.Helper() + c := assert.NewAborting(t) + scheme := runtime.NewScheme() + c.NoError(clientgoscheme.AddToScheme(scheme), "add to scheme") + builder := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&corev1.Pod{}) + for _, p := range pods { + builder = builder.WithObjects(p) + } + f := &simFixture{c: c, base: builder.Build()} + cl := interceptor.NewClient(f.base.(client.WithWatch), interceptor.Funcs{ + SubResourcePatch: func( + ctx context.Context, w client.Client, sub string, obj client.Object, + patch client.Patch, opts ...client.SubResourcePatchOption, + ) error { + f.patches.Add(1) + return w.SubResource(sub).Patch(ctx, obj, patch, opts...) + }, + }) + f.sim = NewDataPlaneSim(cl, time.Hour) + return f +} + +func (f *simFixture) tick() { f.sim.tickPods(context.Background()) } + +func (f *simFixture) get(name string) *corev1.Pod { + f.c.Helper() + p := &corev1.Pod{} + f.c.NoError(f.base.Get(context.Background(), client.ObjectKey{Namespace: "ns", Name: name}, p), + "get pod %s", name) + return p +} + +func condition(p *corev1.Pod, t corev1.PodConditionType) (corev1.ConditionStatus, string) { + for _, c := range p.Status.Conditions { + if c.Type == t { + return c.Status, c.Reason + } + } + return "", "" +} + +func status(p *corev1.Pod, name string) corev1.ContainerStatus { + for _, cs := range append(p.Status.InitContainerStatuses, p.Status.ContainerStatuses...) { + if cs.Name == name { + return cs + } + } + return corev1.ContainerStatus{} +} + +func stateOf(cs corev1.ContainerStatus) string { + switch { + case cs.State.Running != nil: + return "Running" + case cs.State.Waiting != nil: + return "Waiting/" + cs.State.Waiting.Reason + case cs.State.Terminated != nil: + return "Terminated/" + cs.State.Terminated.Reason + } + return "none" +} + +// summary is the part of a pod's status these tests assert on, in one +// comparable string, so a failure shows the whole shape at once. +func summary(p *corev1.Pod) string { + s := string(p.Status.Phase) + for _, t := range []corev1.PodConditionType{ + corev1.PodScheduled, corev1.PodInitialized, corev1.ContainersReady, corev1.PodReady, + } { + st, reason := condition(p, t) + s += fmt.Sprintf(" %s=%s", t, st) + if reason != "" { + s += "(" + reason + ")" + } + } + for _, name := range []string{"setup", "db", "migrate", "app"} { + cs := status(p, name) + s += fmt.Sprintf(" %s:%s", name, stateOf(cs)) + if cs.Ready { + s += "+ready" + } + } + return s +} + +func TestDataPlaneSimPodStatus(t *testing.T) { + cases := []struct { + name string + fault *PodFault + want string + }{ + { + name: "healthy: the sidecar runs and counts toward readiness", + want: "Running PodScheduled=True Initialized=True ContainersReady=True Ready=True" + + " setup:Terminated/Completed+ready db:Running+ready" + + " migrate:Terminated/Completed+ready app:Running+ready", + }, + { + name: "a regular container failing readiness", + fault: &PodFault{NotReady: []string{"app"}}, + want: "Running PodScheduled=True Initialized=True" + + " ContainersReady=False(ContainersNotReady) Ready=False(ContainersNotReady)" + + " setup:Terminated/Completed+ready db:Running+ready" + + " migrate:Terminated/Completed+ready app:Running", + }, + { + name: "a sidecar failing readiness has started, so it blocks nothing", + fault: &PodFault{NotReady: []string{"db"}}, + want: "Running PodScheduled=True Initialized=True" + + " ContainersReady=False(ContainersNotReady) Ready=False(ContainersNotReady)" + + " setup:Terminated/Completed+ready db:Running" + + " migrate:Terminated/Completed+ready app:Running+ready", + }, + { + name: "a crashlooping sidecar holds back everything after it", + fault: &PodFault{CrashLoop: []string{"db"}}, + want: "Pending PodScheduled=True Initialized=False(ContainersNotInitialized)" + + " ContainersReady=False(ContainersNotReady) Ready=False(ContainersNotReady)" + + " setup:Terminated/Completed+ready db:Waiting/CrashLoopBackOff" + + " migrate:Waiting/PodInitializing app:Waiting/PodInitializing", + }, + { + name: "a crashlooping regular init container", + fault: &PodFault{CrashLoop: []string{"setup"}}, + want: "Pending PodScheduled=True Initialized=False(ContainersNotInitialized)" + + " ContainersReady=False(ContainersNotReady) Ready=False(ContainersNotReady)" + + " setup:Waiting/CrashLoopBackOff db:Waiting/PodInitializing" + + " migrate:Waiting/PodInitializing app:Waiting/PodInitializing", + }, + { + name: "a regular init container that never completes", + fault: &PodFault{NotReady: []string{"migrate"}}, + want: "Pending PodScheduled=True Initialized=False(ContainersNotInitialized)" + + " ContainersReady=False(ContainersNotReady) Ready=False(ContainersNotReady)" + + " setup:Terminated/Completed+ready db:Running+ready" + + " migrate:Running app:Waiting/PodInitializing", + }, + { + name: "unscheduled", + fault: &PodFault{Unscheduled: true}, + want: "Pending PodScheduled=False(Unschedulable) Initialized=False" + + " ContainersReady=False(ContainersNotReady) Ready=False(ContainersNotReady)" + + " setup:none db:none migrate:none app:none", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newSimFixture(t, faultPod("p")) + f.sim.SetPodFaults("ns", func(*corev1.Pod) *PodFault { return tc.fault }, t.Errorf) + f.tick() + f.c.Eq(tc.want, summary(f.get("p")), "pod status") + }) + } +} + +// TestDataPlaneSimFaultHeals pins that a fault is re-evaluated every sweep, so +// lifting it heals the pod: the old sim skipped any pod it had already made +// Ready, which would have made a fault a one-way door. +func TestDataPlaneSimFaultHeals(t *testing.T) { + f := newSimFixture(t, faultPod("p")) + f.tick() + f.c.Eq(corev1.ConditionTrue, condReady(f.get("p")), "healthy first") + + f.sim.SetPodFaults("ns", func(*corev1.Pod) *PodFault { + return &PodFault{CrashLoop: []string{"app"}} + }, t.Errorf) + f.tick() + f.c.Eq(corev1.ConditionFalse, condReady(f.get("p")), "broken") + + f.sim.SetPodFaults("ns", nil, nil) + f.tick() + p := f.get("p") + f.c.Eq(corev1.ConditionTrue, condReady(p), "healed") + f.c.Eq(int32(1), status(p, "app").RestartCount, "restarts survive healing, as on kubelet") +} + +func condReady(p *corev1.Pod) corev1.ConditionStatus { + st, _ := condition(p, corev1.PodReady) + return st +} + +// TestDataPlaneSimReadinessGates pins kubelet's Ready rule: containers ready is +// not enough while a readiness gate condition is not True, and the simulator +// never sets a gate itself. +func TestDataPlaneSimReadinessGates(t *testing.T) { + f := newSimFixture(t, faultPod("p", gateCondition)) + f.tick() + p := f.get("p") + st, reason := condition(p, corev1.PodReady) + f.c.Eq(corev1.ConditionFalse, st, "Ready with the gate unset") + f.c.Eq("ReadinessGatesNotReady", reason, "Ready reason with the gate unset") + st, _ = condition(p, corev1.ContainersReady) + f.c.Eq(corev1.ConditionTrue, st, "ContainersReady with the gate unset") + st, _ = condition(p, gateCondition) + f.c.Eq(corev1.ConditionStatus(""), st, "the sim wrote the gate condition itself") + + p.Status.Conditions = append(p.Status.Conditions, corev1.PodCondition{ + Type: gateCondition, + Status: corev1.ConditionTrue, + }) + f.c.NoError(f.base.Status().Update(context.Background(), p), "set gate") + f.tick() + f.c.Eq(corev1.ConditionTrue, condReady(f.get("p")), "Ready once the gate is True") +} + +// TestDataPlaneSimSteadyStateWritesNothing pins that a pod already matching +// what the simulator would report is not patched again, healthy or faulted, +// so the sim adds no churn for RequireQuiescent to trip over. +func TestDataPlaneSimSteadyStateWritesNothing(t *testing.T) { + for _, fault := range []*PodFault{nil, {NotReady: []string{"db"}}, {Unscheduled: true}} { + f := newSimFixture(t, faultPod("p")) + f.sim.SetPodFaults("ns", func(*corev1.Pod) *PodFault { return fault }, t.Errorf) + f.tick() + before := f.patches.Load() + f.c.Eq(int32(1), before, "first sweep patches once (fault %+v)", fault) + f.tick() + f.tick() + f.c.Eq(before, f.patches.Load(), "later sweeps patch nothing (fault %+v)", fault) + } +} + +// TestDataPlaneSimCrashLoopBackoff pins that restarts climb on kubelet's +// backoff rather than once per sweep, which at a quarter-second sweep would +// make a crashlooping pod a write storm. +func TestDataPlaneSimCrashLoopBackoff(t *testing.T) { + f := newSimFixture(t, faultPod("p")) + fault := &PodFault{CrashLoop: []string{"app"}} + + start := metav1.NewTime(time.Unix(1_000_000, 0)) + p := f.get("p") + at := func(d time.Duration) int32 { + p.Status = f.sim.podStatus(p, fault, metav1.NewTime(start.Add(d))) + return status(p, "app").RestartCount + } + f.c.Eq(int32(1), at(0), "entering the crash loop is one restart") + f.c.Eq(int32(1), at(5*time.Second), "inside the first 10s backoff") + f.c.Eq(int32(2), at(10*time.Second), "after the first backoff") + f.c.Eq(int32(2), at(25*time.Second), "inside the second, 20s backoff") + f.c.Eq(int32(3), at(30*time.Second), "after the second backoff") + + for restarts, want := range map[int32]time.Duration{ + 1: 10 * time.Second, 2: 20 * time.Second, 5: 160 * time.Second, 6: 5 * time.Minute, 30: 5 * time.Minute, + } { + f.c.Eq(want, crashLoopBackoff(restarts), "backoff after %d restarts", restarts) + } +} + +type recordedReports struct{ msgs []string } + +func (r *recordedReports) report(format string, args ...any) { + r.msgs = append(r.msgs, fmt.Sprintf(format, args...)) +} + +// TestDataPlaneSimReportsUnknownContainers pins the guard against a fault that +// does nothing because it names a container the pod does not have: reported, +// once per pod and name however many sweeps see it, and never through a rule +// that has since been replaced. +func TestDataPlaneSimReportsUnknownContainers(t *testing.T) { + f := newSimFixture(t, faultPod("p"), faultPod("q")) + var r recordedReports + f.sim.SetPodFaults("ns", func(*corev1.Pod) *PodFault { + return &PodFault{CrashLoop: []string{"app", "postgress"}} + }, r.report) + f.tick() + f.tick() + f.c.Len(r.msgs, 2, "one report per pod: %q", r.msgs) + + var after recordedReports + f.sim.SetPodFaults("ns", nil, nil) + f.sim.SetPodFaults("ns", func(*corev1.Pod) *PodFault { return nil }, after.report) + f.tick() + f.c.Len(r.msgs, 2, "the replaced rule reported again: %q", r.msgs) + f.c.Len(after.msgs, 0, "a healthy rule reported: %q", after.msgs) +} + +// TestFaultPods drives the whole path through the shared suite: a fault +// installed by C.FaultPods reaches the running simulator, matches by what the +// pod runs rather than its name, and heals when replaced. +func TestFaultPods(t *testing.T) { + c := shared.Case(t) + broken := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "broken", Namespace: c.NS}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "app", Image: "bad"}}}, + } + fine := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "fine", Namespace: c.NS}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "app", Image: "good"}}}, + } + c.FaultPods(func(p *corev1.Pod) *PodFault { + if p.Spec.Containers[0].Image == "bad" { + return &PodFault{CrashLoop: []string{"app"}} + } + return nil + }) + c.NoError(c.Create(broken), "create broken pod") + c.NoError(c.Create(fine), "create fine pod") + + readyIs := func(name string, want corev1.ConditionStatus) func() error { + return func() error { + p := &corev1.Pod{} + if err := c.Get(client.ObjectKey{Namespace: c.NS, Name: name}, p); err != nil { + return err + } + if got := condReady(p); got != want { + return fmt.Errorf("pod %s Ready=%q: %s", name, got, summary(p)) + } + return nil + } + } + c.Eventually(10*time.Second, "fine pod Ready", readyIs("fine", corev1.ConditionTrue)) + c.Eventually(10*time.Second, "broken pod not Ready", readyIs("broken", corev1.ConditionFalse)) + + c.FaultPods(nil) + c.Eventually(10*time.Second, "broken pod healed", readyIs("broken", corev1.ConditionTrue)) +} diff --git a/ctrltest/suite.go b/ctrltest/suite.go index 2135b68..f75d3f8 100644 --- a/ctrltest/suite.go +++ b/ctrltest/suite.go @@ -176,6 +176,10 @@ type Suite struct { managers []*Manager byName map[string]*Manager + // sim is the data plane simulator, nil when Options.SimInterval is not + // positive. Kept so C.FaultPods can reach it. + sim *DataPlaneSim + // simDone closes when the data plane simulator's goroutine has returned. // Nil when no simulator was started. simDone chan struct{} @@ -376,10 +380,11 @@ func Boot(opts Options) (*Suite, func() error, error) { // looked, which is the shape of an intermittent CI failure rather than of // a clean shutdown. if opts.SimInterval > 0 { + s.sim = NewDataPlaneSim(s.Client, opts.SimInterval) s.simDone = make(chan struct{}) go func() { defer close(s.simDone) - NewDataPlaneSim(s.Client, opts.SimInterval).Run(ctx) + s.sim.Run(ctx) }() }