diff --git a/controllers/clustersummary_controller.go b/controllers/clustersummary_controller.go index 148f25ec..50daa2bb 100644 --- a/controllers/clustersummary_controller.go +++ b/controllers/clustersummary_controller.go @@ -57,6 +57,7 @@ import ( libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/deployer" + "github.com/projectsveltos/libsveltos/lib/eventreasons" license "github.com/projectsveltos/libsveltos/lib/licenses" logs "github.com/projectsveltos/libsveltos/lib/logsettings" predicates "github.com/projectsveltos/libsveltos/lib/predicates" @@ -607,23 +608,13 @@ func (r *ClusterSummaryReconciler) prepareForDeployment(ctx context.Context, func (r *ClusterSummaryReconciler) proceedDeployingClusterSummary(ctx context.Context, clusterSummaryScope *scope.ClusterSummaryScope, logger logr.Logger) (reconcile.Result, error) { - // Snapshot existing failure messages before deploying so we can detect - // new conflicts vs. ongoing ones and avoid re-raising an event every retry. - preDeployFailures := collectFailureMessages(clusterSummaryScope.ClusterSummary) - err := r.deploy(ctx, clusterSummaryScope, logger) if err != nil { var conflictErr *deployer.ConflictError ok := errors.As(err, &conflictErr) if ok { + // The ProfileConflict event is recorded by updateFeatureStatus, when the conflict is persisted logger.V(logs.LogInfo).Error(err, "failed to deploy because of conflict") - if _, alreadyKnown := preDeployFailures[conflictErr.Error()]; !alreadyKnown { - clusterSummary := clusterSummaryScope.ClusterSummary - r.eventRecorder.Eventf(clusterSummary, nil, corev1.EventTypeWarning, "Conflict", - configv1beta1.ClusterSummaryKind, "Conflict detected for cluster %s %s/%s: %s", - clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, - clusterSummary.Spec.ClusterName, conflictErr.Error()) - } r.setNextReconcileTime(clusterSummaryScope, r.ConflictRetryTime) return reconcile.Result{RequeueAfter: r.ConflictRetryTime}, nil } @@ -2143,14 +2134,23 @@ func grantsPullModeEligibility(result *license.LicenseVerificationResult, func (r *ClusterSummaryReconciler) updateStatusWithMissingLicenseError( clusterSummaryScope *scope.ClusterSummaryScope, logger logr.Logger) { - notEligibleError := errors.New("license is required to manage SveltosCluster in pull mode") + // All features fail for the same reason: one event, recorded the first time + _, previousMessage := getFeatureFailure(clusterSummaryScope.ClusterSummary, libsveltosv1beta1.FeatureHelm) + failed := libsveltosv1beta1.FeatureStatusFailed r.updateFeatureStatus(clusterSummaryScope, libsveltosv1beta1.FeatureHelm, &failed, nil, - notEligibleError, logger) + errPullModeLicenseRequired, logger) r.updateFeatureStatus(clusterSummaryScope, libsveltosv1beta1.FeatureKustomize, &failed, nil, - notEligibleError, logger) + errPullModeLicenseRequired, logger) r.updateFeatureStatus(clusterSummaryScope, libsveltosv1beta1.FeatureResources, &failed, nil, - notEligibleError, logger) + errPullModeLicenseRequired, logger) + + if previousMessage == nil || *previousMessage != errPullModeLicenseRequired.Error() { + clusterSummary := clusterSummaryScope.ClusterSummary + recordEventf(r.eventRecorder, clusterSummary, nil, corev1.EventTypeWarning, eventreasons.LicenseInvalid, + eventreasons.ActionDeploying, "Cluster %s %s/%s: %s", clusterSummary.Spec.ClusterType, + clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, errPullModeLicenseRequired.Error()) + } } // removeFinalizer removes the finalizer @@ -2272,16 +2272,3 @@ func getClusterSummaryWithInstantiatedCharts(ctx context.Context, cs *configv1be return &csCopy, nil } - -// collectFailureMessages returns the set of FailureMessages currently recorded -// across all FeatureSummaries. Used to detect whether a conflict is new (not -// yet in the status) versus ongoing (already reported in a previous reconcile). -func collectFailureMessages(cs *configv1beta1.ClusterSummary) map[string]struct{} { - msgs := make(map[string]struct{}, len(cs.Status.FeatureSummaries)) - for i := range cs.Status.FeatureSummaries { - if msg := cs.Status.FeatureSummaries[i].FailureMessage; msg != nil { - msgs[*msg] = struct{}{} - } - } - return msgs -} diff --git a/controllers/clustersummary_deployer.go b/controllers/clustersummary_deployer.go index 4fcc912c..dc0d3783 100644 --- a/controllers/clustersummary_deployer.go +++ b/controllers/clustersummary_deployer.go @@ -37,6 +37,7 @@ import ( "github.com/projectsveltos/libsveltos/lib/clustercache" "github.com/projectsveltos/libsveltos/lib/clusterproxy" "github.com/projectsveltos/libsveltos/lib/deployer" + "github.com/projectsveltos/libsveltos/lib/eventreasons" logs "github.com/projectsveltos/libsveltos/lib/logsettings" "github.com/projectsveltos/libsveltos/lib/pullmode" ) @@ -179,8 +180,8 @@ func (r *ClusterSummaryReconciler) proceedDeployingFeature(ctx context.Context, clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName) cluster, _ := clusterproxy.GetCluster(ctx, getManagementClusterClient(), clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, clusterSummary.Spec.ClusterType) - r.eventRecorder.Eventf(clusterSummary, cluster, corev1.EventTypeNormal, "Sveltos", - configv1beta1.ClusterSummaryKind, message) + recordEventf(r.eventRecorder, clusterSummary, cluster, corev1.EventTypeNormal, eventreasons.FeatureDeployed, + featureAction(eventreasons.ActionDeploying, f.id), "%s", message) return nil } @@ -251,8 +252,8 @@ func (r *ClusterSummaryReconciler) handleDeployerError(deployerError error, clus if errors.As(deployerError, &nonRetriableError) { nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, deployerError, logger) - r.eventRecorder.Eventf(clusterSummary, nil, corev1.EventTypeWarning, "FailedNonRetriable", - configv1beta1.ClusterSummaryKind, + recordEventf(r.eventRecorder, clusterSummary, nil, corev1.EventTypeWarning, eventreasons.FeatureFailedNonRetriable, + featureAction(eventreasons.ActionDeploying, f.id), "Feature %s for cluster %s %s/%s failed with non-retriable error: %s", f.id, clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, deployerError.Error()) @@ -262,8 +263,8 @@ func (r *ClusterSummaryReconciler) handleDeployerError(deployerError error, clus if errors.As(deployerError, &templateError) { nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, deployerError, logger) - r.eventRecorder.Eventf(clusterSummary, nil, corev1.EventTypeWarning, "FailedNonRetriable", - configv1beta1.ClusterSummaryKind, + recordEventf(r.eventRecorder, clusterSummary, nil, corev1.EventTypeWarning, eventreasons.FeatureFailedNonRetriable, + featureAction(eventreasons.ActionDeploying, f.id), "Feature %s for cluster %s %s/%s failed to instantiate template: %s", f.id, clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, deployerError.Error()) @@ -327,8 +328,8 @@ func (r *ClusterSummaryReconciler) handlePullModeProvisioned(ctx context.Context clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, clusterSummary.Spec.ClusterType) message := fmt.Sprintf("Feature: %s deployed to cluster %s %s/%s", f.id, clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName) - r.eventRecorder.Eventf(clusterSummary, cluster, corev1.EventTypeNormal, "Sveltos", - configv1beta1.ClusterSummaryKind, message) + recordEventf(r.eventRecorder, clusterSummary, cluster, corev1.EventTypeNormal, eventreasons.FeatureDeployed, + featureAction(eventreasons.ActionDeploying, f.id), "%s", message) r.updateFeatureStatus(clusterSummaryScope, f.id, pullmodeProvisionedStatus(), currentHash, nil, logger) clusterSummaryScope.SetFailureMessage(f.id, nil) now := metav1.NewTime(time.Now()) @@ -610,10 +611,15 @@ func (r *ClusterSummaryReconciler) processUndeployResult(ctx context.Context, cl func (r *ClusterSummaryReconciler) markUndeployFailureRetriable(clusterSummaryScope *scope.ClusterSummaryScope, featureID libsveltosv1beta1.FeatureID, failureErr error, logger logr.Logger) { + // What the ClusterSummary recorded for this feature before this update tells whether this failure is new + _, previousMessage := getFeatureFailure(clusterSummaryScope.ClusterSummary, featureID) + removing := libsveltosv1beta1.FeatureStatusRemoving r.updateFeatureStatus(clusterSummaryScope, featureID, &removing, nil, failureErr, logger) errorMsg := failureErr.Error() clusterSummaryScope.SetFailureMessage(featureID, &errorMsg) + r.recordFeatureFailure(clusterSummaryScope.ClusterSummary, featureID, eventreasons.ActionRemoving, + previousMessage, failureErr) } func cleanPreviousActiveTracking(ctx context.Context, clusterSummary *configv1beta1.ClusterSummary, f feature, @@ -956,11 +962,16 @@ func (r *ClusterSummaryReconciler) updateFeatureStatus(clusterSummaryScope *scop // non-terminal statuses above in case that ever changes. clusterSummaryScope.SetFeatureStatus(featureID, libsveltosv1beta1.FeatureStatusBlocked, hash, nil) case libsveltosv1beta1.FeatureStatusFailed, libsveltosv1beta1.FeatureStatusFailedNonRetriable: + // What the ClusterSummary recorded for this feature before this update tells whether this failure is new + previousStatus, previousMessage := getFeatureFailure(clusterSummary, featureID) failed := true clusterSummaryScope.SetFeatureStatus(featureID, *status, hash, &failed) if statusError != nil { err := statusError.Error() clusterSummaryScope.SetFailureMessage(featureID, &err) + if *status == libsveltosv1beta1.FeatureStatusFailed { + r.recordFailedStatus(clusterSummary, featureID, previousStatus, previousMessage, statusError) + } } clusterSummaryScope.SetLastAppliedTime(featureID, &now) trackReconcileOutcome(clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, @@ -1087,8 +1098,8 @@ func (r *ClusterSummaryReconciler) markMaxConsecutiveFailuresReached(clusterSumm nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable resultError := errors.New("the maximum number of consecutive errors has been reached") r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, resultError, logger) - r.eventRecorder.Eventf(clusterSummary, nil, corev1.EventTypeWarning, "FailedNonRetriable", - configv1beta1.ClusterSummaryKind, + recordEventf(r.eventRecorder, clusterSummary, nil, corev1.EventTypeWarning, eventreasons.FeatureFailedNonRetriable, + featureAction(eventreasons.ActionDeploying, f.id), "Feature %s for cluster %s %s/%s will no longer be retried: maximum consecutive failures reached", f.id, clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName) diff --git a/controllers/events.go b/controllers/events.go new file mode 100644 index 00000000..9acaf11d --- /dev/null +++ b/controllers/events.go @@ -0,0 +1,169 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/events" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/deployer" + "github.com/projectsveltos/libsveltos/lib/eventreasons" +) + +// recordEventf records an event on regarding, about the related object (can be nil). +// It does nothing when no recorder is set. +// The note is capped, so an oversized message is not rejected by the API server. +func recordEventf(recorder events.EventRecorder, regarding, related runtime.Object, eventType, reason, action, + format string, args ...any) { + + if recorder == nil { + return + } + + note := eventreasons.TruncateNote(fmt.Sprintf(format, args...)) + recorder.Eventf(regarding, related, eventType, reason, action, "%s", note) +} + +// featureAction returns the action of an event about a feature: the verb followed by the feature, for +// instance DeployingHelm. The events recorder merges events with the same reason, action and regarding +// object and keeps only the first note, so an action which is the same for every feature would make +// the failures of two features of the same ClusterSummary a single event. +func featureAction(action string, featureID libsveltosv1beta1.FeatureID) string { + return action + string(featureID) +} + +// getActionForFeatureStatus returns the action an event refers to given the status the feature had before: +// a feature being removed is Removing, anything else is Deploying. +func getActionForFeatureStatus(previousStatus *libsveltosv1beta1.FeatureStatus) string { + if previousStatus != nil && + (*previousStatus == libsveltosv1beta1.FeatureStatusRemoving || + *previousStatus == libsveltosv1beta1.FeatureStatusAgentRemoving) { + + return eventreasons.ActionRemoving + } + + return eventreasons.ActionDeploying +} + +var ( + // errPullModeLicenseRequired is the failure of a ClusterSummary targeting a SveltosCluster in pull mode which + // is not covered by a license. + errPullModeLicenseRequired = errors.New("license is required to manage SveltosCluster in pull mode") +) + +// hasOwnEvent returns true for the errors which are already reported by their own event: a failure which +// will not be retried (FeatureFailedNonRetriable) and a missing license (LicenseInvalid). +// FeatureFailed is not recorded for those, or the same failure would be reported twice. +func hasOwnEvent(err error) bool { + var nonRetriableError *configv1beta1.NonRetriableError + var templateError *configv1beta1.TemplateInstantiationError + + return errors.As(err, &nonRetriableError) || + errors.As(err, &templateError) || + errors.Is(err, errPullModeLicenseRequired) +} + +// isConflictError returns true when err is, or wraps, a conflict with another profile. +func isConflictError(err error) bool { + var conflictError *deployer.ConflictError + + return errors.As(err, &conflictError) +} + +// recordFailedStatus records the event for a feature which has just been set to Failed with failure: a +// ProfileConflict for a conflict with another profile, a FeatureFailed for any other failure which is +// retried. The failures which have their own event (hasOwnEvent) are not recorded here. +// previousStatus and previousMessage are what the ClusterSummary recorded before this update. +func (r *ClusterSummaryReconciler) recordFailedStatus(clusterSummary *configv1beta1.ClusterSummary, + featureID libsveltosv1beta1.FeatureID, previousStatus *libsveltosv1beta1.FeatureStatus, previousMessage *string, + failure error) { + + switch { + case isConflictError(failure): + r.recordProfileConflict(clusterSummary, previousMessage, failure) + case !hasOwnEvent(failure): + r.recordFeatureFailure(clusterSummary, featureID, getActionForFeatureStatus(previousStatus), + previousMessage, failure) + } +} + +// recordProfileConflict records a Warning ProfileConflict event: a resource this ClusterSummary wants to deploy +// is already deployed by another profile. Nothing is recorded when the ClusterSummary already failed with the +// same message: a retry which hits the same conflict is not news. +func (r *ClusterSummaryReconciler) recordProfileConflict(clusterSummary *configv1beta1.ClusterSummary, + previousMessage *string, conflict error) { + + message := conflict.Error() + if previousMessage != nil && *previousMessage == message { + return + } + + recordEventf(r.eventRecorder, clusterSummary, nil, corev1.EventTypeWarning, eventreasons.ProfileConflict, + eventreasons.ActionDeploying, "Conflict detected for cluster %s %s/%s: %s", + clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, + clusterSummary.Spec.ClusterName, message) +} + +// getFeatureFailure returns the status and the failure message the ClusterSummary currently records for +// the feature. Both are nil when the feature has no entry yet. +// Call it before updating the feature status: the failure message of the previous attempt is what tells +// whether a failure is new. +func getFeatureFailure(clusterSummary *configv1beta1.ClusterSummary, featureID libsveltosv1beta1.FeatureID, +) (status *libsveltosv1beta1.FeatureStatus, message *string) { + + featureSummary := getFeatureSummaryForFeatureID(clusterSummary, featureID) + if featureSummary == nil { + return nil, nil + } + + currentStatus := featureSummary.Status + status = ¤tStatus + if featureSummary.FailureMessage != nil { + currentMessage := *featureSummary.FailureMessage + message = ¤tMessage + } + + return status, message +} + +// recordFeatureFailure records a Warning FeatureFailed event when a feature fails to be deployed, or removed, +// and the operation is going to be retried. Nothing is recorded when the feature already failed with the same +// message (previousMessage): a retry which fails the same way is not news. A different message is recorded. +// The failure message is cleared when the feature succeeds, so a failure after a success is recorded again. +func (r *ClusterSummaryReconciler) recordFeatureFailure(clusterSummary *configv1beta1.ClusterSummary, + featureID libsveltosv1beta1.FeatureID, action string, previousMessage *string, failure error) { + + if failure == nil { + return + } + + message := failure.Error() + if previousMessage != nil && *previousMessage == message { + return + } + + recordEventf(r.eventRecorder, clusterSummary, nil, corev1.EventTypeWarning, eventreasons.FeatureFailed, + featureAction(action, featureID), "Feature %s for cluster %s %s/%s failed: %s", + featureID, clusterSummary.Spec.ClusterType, clusterSummary.Spec.ClusterNamespace, + clusterSummary.Spec.ClusterName, message) +} diff --git a/controllers/events_test.go b/controllers/events_test.go new file mode 100644 index 00000000..4bf7554d --- /dev/null +++ b/controllers/events_test.go @@ -0,0 +1,355 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "context" + "errors" + "fmt" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/events" + "k8s.io/klog/v2/textlogger" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/controllers" + "github.com/projectsveltos/addon-controller/lib/clusterops" + "github.com/projectsveltos/addon-controller/pkg/scope" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/deployer" + fakedeployer "github.com/projectsveltos/libsveltos/lib/deployer/fake" +) + +const eventBufferSize = 20 + +var _ = Describe("ClusterSummary events", func() { + var logger logr.Logger + var clusterProfile *configv1beta1.ClusterProfile + var clusterSummary *configv1beta1.ClusterSummary + var cluster *clusterv1.Cluster + var namespace string + var recorder *events.FakeRecorder + + BeforeEach(func() { + namespace = randomString() + logger = textlogger.NewLogger(textlogger.NewConfig()) + + clusterProfile = &configv1beta1.ClusterProfile{ + ObjectMeta: metav1.ObjectMeta{Name: clusterProfileNamePrefix + randomString()}, + Spec: configv1beta1.Spec{ + ClusterSelector: libsveltosv1beta1.Selector{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{randomString(): randomString()}}, + }, + }, + } + + clusterName := randomString() + clusterSummary = &configv1beta1.ClusterSummary{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterops.GetClusterSummaryName(configv1beta1.ClusterProfileKind, + clusterProfile.Name, clusterName, false), + Namespace: namespace, + }, + Spec: configv1beta1.ClusterSummarySpec{ + ClusterNamespace: namespace, + ClusterName: clusterName, + ClusterType: libsveltosv1beta1.ClusterTypeCapi, + }, + } + addLabelsToClusterSummary(clusterSummary, clusterProfile.Name, clusterName, libsveltosv1beta1.ClusterTypeCapi) + + cluster = &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: clusterName}, + } + + recorder = events.NewFakeRecorder(eventBufferSize) + }) + + // newClient returns a client with the objects of the test + newClient := func() client.Client { + initObjects := []client.Object{clusterSummary, clusterProfile, cluster} + return fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(initObjects...). + WithObjects(initObjects...).Build() + } + + // newReconciler returns a reconciler recording its events in recorder, and a scope on the ClusterSummary. + newReconciler := func(c client.Client, dep deployer.DeployerInterface, + ) (*controllers.ClusterSummaryReconciler, *scope.ClusterSummaryScope) { + + reconciler := getClusterSummaryReconciler(c, dep) + controllers.SetEventRecorder(reconciler, recorder) + + return reconciler, getClusterSummaryScope(c, logger, clusterProfile, clusterSummary) + } + + notePrefix := func(featureID libsveltosv1beta1.FeatureID) string { + return fmt.Sprintf("Feature %s for cluster Capi %s/%s failed: ", featureID, namespace, clusterSummary.Spec.ClusterName) + } + + Context("failures which are retried", func() { + It("records FeatureFailed once, and again when the failure message changes", func() { + reconciler, clusterSummaryScope := newReconciler(newClient(), nil) + hash := []byte(randomString()) + + failed := libsveltosv1beta1.FeatureStatusFailed + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureResources, &failed, + hash, errors.New("boom"), logger) + Expect(recorder.Events).To(Receive(Equal("Warning FeatureFailed " + notePrefix(libsveltosv1beta1.FeatureResources) + "boom"))) + + // A retry cycle goes through Provisioning and fails the same way: not new + provisioning := libsveltosv1beta1.FeatureStatusProvisioning + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureResources, &provisioning, + hash, nil, logger) + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureResources, &failed, + hash, errors.New("boom"), logger) + Expect(recorder.Events).To(BeEmpty()) + + // Different failure + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureResources, &failed, + hash, errors.New("another failure"), logger) + Expect(recorder.Events).To(Receive(Equal("Warning FeatureFailed " + notePrefix(libsveltosv1beta1.FeatureResources) + + "another failure"))) + }) + + It("records a failure again after the feature succeeded", func() { + reconciler, clusterSummaryScope := newReconciler(newClient(), nil) + hash := []byte(randomString()) + + failed := libsveltosv1beta1.FeatureStatusFailed + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureHelm, &failed, + hash, errors.New("boom"), logger) + Expect(recorder.Events).To(Receive(ContainSubstring("Warning FeatureFailed"))) + + provisioned := libsveltosv1beta1.FeatureStatusProvisioned + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureHelm, &provisioned, + hash, nil, logger) + + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureHelm, &failed, + hash, errors.New("boom"), logger) + Expect(recorder.Events).To(Receive(ContainSubstring("Warning FeatureFailed"))) + }) + + It("uses a different action for each feature, so the events of two features are not merged", func() { + recorder.Verbose = true + reconciler, clusterSummaryScope := newReconciler(newClient(), nil) + hash := []byte(randomString()) + + failed := libsveltosv1beta1.FeatureStatusFailed + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureHelm, &failed, + hash, errors.New("boom"), logger) + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureResources, &failed, + hash, errors.New("boom"), logger) + + Expect(recorder.Events).To(Receive(ContainSubstring("FeatureFailed DeployingHelm"))) + Expect(recorder.Events).To(Receive(ContainSubstring("FeatureFailed DeployingResources"))) + }) + + It("is recorded when deployFeature gets a failure from the deployer, once for the same failure", func() { + c := newClient() + dep := fakedeployer.GetClient(context.TODO(), logger, c) + reconciler, clusterSummaryScope := newReconciler(c, dep) + + f := controllers.GetHandlersForFeature(libsveltosv1beta1.FeatureResources) + hash, err := controllers.ResourcesHash(context.TODO(), c, clusterSummary, logger) + Expect(err).To(BeNil()) + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureResources, Hash: hash, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + dep.StoreResult(clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, clusterSummary.Name, + string(libsveltosv1beta1.FeatureResources), libsveltosv1beta1.ClusterTypeCapi, false, + errors.New("failed to apply resources")) + + Expect(controllers.DeployFeature(reconciler, context.TODO(), clusterSummaryScope, f, logger)).ToNot(BeNil()) + Expect(recorder.Events).To(Receive(Equal("Warning FeatureFailed " + notePrefix(libsveltosv1beta1.FeatureResources) + + "failed to apply resources"))) + + Expect(controllers.DeployFeature(reconciler, context.TODO(), clusterSummaryScope, f, logger)).ToNot(BeNil()) + Expect(recorder.Events).To(BeEmpty()) + }) + }) + + Context("failures reported by their own event", func() { + It("does not record FeatureFailed for a failure which will not be retried or a template error", func() { + reconciler, clusterSummaryScope := newReconciler(newClient(), nil) + hash := []byte(randomString()) + + failed := libsveltosv1beta1.FeatureStatusFailed + for _, failure := range []error{ + &configv1beta1.NonRetriableError{Message: "not retriable"}, + &configv1beta1.TemplateInstantiationError{Message: "bad template"}, + } { + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureHelm, &failed, + hash, failure, logger) + } + Expect(recorder.Events).To(BeEmpty()) + }) + + It("records ProfileConflict, and not FeatureFailed, for a conflict with another profile, once for the same conflict", func() { + reconciler, clusterSummaryScope := newReconciler(newClient(), nil) + hash := []byte(randomString()) + + // The conflict is wrapped, as the handlers return it + conflict := fmt.Errorf("processing ConfigMap default/cm: %w", + deployer.NewConflictError("resource is deployed by another profile")) + + failed := libsveltosv1beta1.FeatureStatusFailed + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureResources, &failed, + hash, conflict, logger) + Expect(recorder.Events).To(Receive(Equal("Warning ProfileConflict Conflict detected for cluster Capi " + + namespace + "/" + clusterSummary.Spec.ClusterName + ": " + conflict.Error()))) + Expect(recorder.Events).To(BeEmpty()) + + // A retry goes through Provisioning and hits the same conflict: not new + provisioning := libsveltosv1beta1.FeatureStatusProvisioning + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureResources, &provisioning, + hash, nil, logger) + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureResources, &failed, + hash, conflict, logger) + Expect(recorder.Events).To(BeEmpty()) + }) + + It("records ProfileConflict when deployFeature gets a conflict from the deployer", func() { + c := newClient() + dep := fakedeployer.GetClient(context.TODO(), logger, c) + reconciler, clusterSummaryScope := newReconciler(c, dep) + + f := controllers.GetHandlersForFeature(libsveltosv1beta1.FeatureResources) + hash, err := controllers.ResourcesHash(context.TODO(), c, clusterSummary, logger) + Expect(err).To(BeNil()) + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureResources, Hash: hash, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + // The conflict is detected in the deployer worker and comes back as the result of the deployment, + // deployFeature does not return it + dep.StoreResult(clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, clusterSummary.Name, + string(libsveltosv1beta1.FeatureResources), libsveltosv1beta1.ClusterTypeCapi, false, + deployer.NewConflictError("resource is deployed by another profile")) + + Expect(controllers.DeployFeature(reconciler, context.TODO(), clusterSummaryScope, f, logger)).ToNot(BeNil()) + Expect(recorder.Events).To(Receive(ContainSubstring("Warning ProfileConflict Conflict detected for cluster Capi " + + namespace + "/" + clusterSummary.Spec.ClusterName))) + Expect(recorder.Events).To(BeEmpty()) + }) + + It("records FeatureFailedNonRetriable, and not FeatureFailed, when deployFeature gets a non retriable error", func() { + c := newClient() + dep := fakedeployer.GetClient(context.TODO(), logger, c) + reconciler, clusterSummaryScope := newReconciler(c, dep) + + f := controllers.GetHandlersForFeature(libsveltosv1beta1.FeatureResources) + hash, err := controllers.ResourcesHash(context.TODO(), c, clusterSummary, logger) + Expect(err).To(BeNil()) + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureResources, Hash: hash, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + dep.StoreResult(clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, clusterSummary.Name, + string(libsveltosv1beta1.FeatureResources), libsveltosv1beta1.ClusterTypeCapi, false, + &configv1beta1.NonRetriableError{Message: "invalid configuration"}) + + _ = controllers.DeployFeature(reconciler, context.TODO(), clusterSummaryScope, f, logger) + Expect(recorder.Events).To(Receive(Equal("Warning FeatureFailedNonRetriable Feature Resources for cluster Capi " + + namespace + "/" + clusterSummary.Spec.ClusterName + " failed with non-retriable error: invalid configuration"))) + Expect(recorder.Events).To(BeEmpty()) + }) + }) + + Context("feature deployed", func() { + It("records a Normal FeatureDeployed event", func() { + c := newClient() + dep := fakedeployer.GetClient(context.TODO(), logger, c) + reconciler, clusterSummaryScope := newReconciler(c, dep) + + f := controllers.GetHandlersForFeature(libsveltosv1beta1.FeatureResources) + hash, err := controllers.ResourcesHash(context.TODO(), c, clusterSummary, logger) + Expect(err).To(BeNil()) + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureResources, Hash: hash, Status: libsveltosv1beta1.FeatureStatusProvisioning}, + } + + dep.StoreResult(clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, clusterSummary.Name, + string(libsveltosv1beta1.FeatureResources), libsveltosv1beta1.ClusterTypeCapi, false, nil) + + Expect(controllers.DeployFeature(reconciler, context.TODO(), clusterSummaryScope, f, logger)).To(BeNil()) + Expect(recorder.Events).To(Receive(Equal("Normal FeatureDeployed Feature: Resources deployed to cluster Capi " + + namespace + "/" + clusterSummary.Spec.ClusterName))) + }) + }) + + Context("failures when a feature is removed", func() { + It("records FeatureFailed with the action Removing when the undeploy fails, once for the same failure", func() { + recorder.Verbose = true + clusterSummary.Status.FeatureSummaries = []configv1beta1.FeatureSummary{ + {FeatureID: libsveltosv1beta1.FeatureResources, Status: libsveltosv1beta1.FeatureStatusRemoving}, + } + + c := newClient() + dep := fakedeployer.GetClient(context.TODO(), logger, c) + reconciler, clusterSummaryScope := newReconciler(c, dep) + + denial := apierrors.NewForbidden(schema.GroupResource{Group: "example.io", Resource: "widgets"}, + "external", errors.New("admission webhook denied the request: cannot delete the widget")) + dep.StoreResult(clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, clusterSummary.Name, + string(libsveltosv1beta1.FeatureResources), libsveltosv1beta1.ClusterTypeCapi, true, denial) + + f := controllers.GetHandlersForFeature(libsveltosv1beta1.FeatureResources) + Expect(controllers.UndeployFeature(reconciler, context.TODO(), clusterSummaryScope, f, logger)).To(BeNil()) + Expect(recorder.Events).To(Receive(And( + ContainSubstring("FeatureFailed RemovingResources"), + ContainSubstring(denial.Error())))) + + Expect(controllers.UndeployFeature(reconciler, context.TODO(), clusterSummaryScope, f, logger)).To(BeNil()) + Expect(recorder.Events).To(BeEmpty()) + }) + }) + + Context("pull mode without a license", func() { + It("records a single LicenseInvalid event, and no FeatureFailed, for the three features", func() { + reconciler, clusterSummaryScope := newReconciler(newClient(), nil) + + controllers.UpdateStatusWithMissingLicenseError(reconciler, clusterSummaryScope, logger) + Expect(recorder.Events).To(Receive(Equal("Warning LicenseInvalid Cluster Capi " + namespace + "/" + + clusterSummary.Spec.ClusterName + ": license is required to manage SveltosCluster in pull mode"))) + Expect(recorder.Events).To(BeEmpty()) + + // Still not licensed: not new + controllers.UpdateStatusWithMissingLicenseError(reconciler, clusterSummaryScope, logger) + Expect(recorder.Events).To(BeEmpty()) + }) + }) + + It("does not fail when no recorder is set", func() { + reconciler, clusterSummaryScope := newReconciler(newClient(), nil) + controllers.SetEventRecorder(reconciler, nil) + + failed := libsveltosv1beta1.FeatureStatusFailed + Expect(func() { + controllers.UpdateFeatureStatus(reconciler, clusterSummaryScope, libsveltosv1beta1.FeatureHelm, &failed, + []byte(randomString()), errors.New("boom"), logger) + }).ToNot(Panic()) + }) +}) diff --git a/controllers/export_test.go b/controllers/export_test.go index 4975fd3f..55f575db 100644 --- a/controllers/export_test.go +++ b/controllers/export_test.go @@ -76,6 +76,7 @@ var ( IsFeatureFailedWithNonRetriableError = (*ClusterSummaryReconciler).isFeatureFailedWithNonRetriableError GetHash = (*ClusterSummaryReconciler).getHash UpdateFeatureStatus = (*ClusterSummaryReconciler).updateFeatureStatus + UpdateStatusWithMissingLicenseError = (*ClusterSummaryReconciler).updateStatusWithMissingLicenseError DeployFeature = (*ClusterSummaryReconciler).deployFeature UndeployFeature = (*ClusterSummaryReconciler).undeployFeature GetCurrentReferences = (*ClusterSummaryReconciler).getCurrentReferences diff --git a/go.mod b/go.mod index dd85b9a3..06380685 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/onsi/gomega v1.44.0 github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.16.1-0.20261010130719-c6f30f77e62f + github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69 github.com/prometheus/client_golang v1.25.0 github.com/sigstore/cosign/v3 v3.1.3 github.com/sigstore/sigstore v1.11.0 diff --git a/go.sum b/go.sum index b6b78274..c1c4d0c0 100644 --- a/go.sum +++ b/go.sum @@ -643,8 +643,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/projectsveltos/libsveltos v1.16.1-0.20261010130719-c6f30f77e62f h1:4+dAshhv+rdzFIlo6q5+3/8mSmxwiqxN1gxl5czeeF8= -github.com/projectsveltos/libsveltos v1.16.1-0.20261010130719-c6f30f77e62f/go.mod h1:wup25BAf1SHuG6FsY8D+tnm1U3/3NiuzcKec/In/fB4= +github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69 h1:rg2W4If7Ejm/3OOLeGiu8GywpK8nFEOfbTsFdVHG+Ho= +github.com/projectsveltos/libsveltos v1.16.1-0.20261011133909-b577d6e6fe69/go.mod h1:wup25BAf1SHuG6FsY8D+tnm1U3/3NiuzcKec/In/fB4= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos=