Debounce the degraded status for flaky conditions

Author: dlom

pkg/operator/credentialsrequest/credentialsrequest_controller.go

@@ -594,9 +594,10 @@ var _ reconcile.Reconciler = &ReconcileCredentialsRequest{}
 // ReconcileCredentialsRequest reconciles a CredentialsRequest object
 type ReconcileCredentialsRequest struct {
 	client.Client
-	AdminClient  client.Client
-	Actuator     actuator.Actuator
-	platformType configv1.PlatformType
+	AdminClient       client.Client
+	Actuator          actuator.Actuator
+	platformType      configv1.PlatformType
+	degradedDebouncer utils.DegradedDebouncer
 }
 
 // Reconcile reads that state of the cluster for a CredentialsRequest object and

pkg/operator/credentialsrequest/status.go

@@ -39,6 +39,7 @@ func (r *ReconcileCredentialsRequest) GetConditions(logger log.FieldLogger) ([]c
 		mode,
 		credRequests,
 		r.platformType,
+		&r.degradedDebouncer,
 		logger), nil
 }
 
@@ -92,6 +93,7 @@ func computeStatusConditions(
 	mode operatorv1.CloudCredentialsMode,
 	credRequests []minterv1.CredentialsRequest,
 	clusterCloudPlatform configv1.PlatformType,
+	degradedDebouncer *utils.DegradedDebouncer,
 	logger log.FieldLogger) []configv1.ClusterOperatorStatusCondition {
 	operatorIsDisabled := mode == operatorv1.CloudCredentialsModeManual
 
@@ -142,14 +144,26 @@ func computeStatusConditions(
 	}
 
 	if failingCredRequests > 0 {
-		var degradedCondition configv1.ClusterOperatorStatusCondition
-		degradedCondition.Type = configv1.OperatorDegraded
-		degradedCondition.Status = configv1.ConditionTrue
-		degradedCondition.Reason = reasonCredentialsFailing
-		degradedCondition.Message = fmt.Sprintf(
-			"%d of %d credentials requests are failing to sync.",
-			failingCredRequests, len(validCredRequests))
-		conditions = append(conditions, degradedCondition)
+		degradedDebouncer.MarkDegraded()
+		if degradedDebouncer.ShouldReport() {
+			var degradedCondition configv1.ClusterOperatorStatusCondition
+			degradedCondition.Type = configv1.OperatorDegraded
+			degradedCondition.Status = configv1.ConditionTrue
+			degradedCondition.Reason = reasonCredentialsFailing
+			degradedCondition.Message = fmt.Sprintf(
+				"%d of %d credentials requests are failing to sync.",
+				failingCredRequests, len(validCredRequests))
+			conditions = append(conditions, degradedCondition)
+		} else {
+			logger.Debugf(
+				"%d of %d credentials are failing to sync, but only for %s",
+				failingCredRequests,
+				len(validCredRequests),
+				degradedDebouncer.DegradedFor(),
+			)
+		}
+	} else {
+		degradedDebouncer.MarkNotDegraded()
 	}
 
 	// Progressing should be true if the operator is making changes to the operand. In this case

pkg/operator/podidentity/podidentitywebhook_controller.go

@@ -285,6 +285,7 @@ type staticResourceReconciler struct {
 	conditions           []configv1.ClusterOperatorStatusCondition
 	cache                resourceapply.ResourceCache
 	podIdentityType      PodIdentityManifestSource
+	degradedDebouncer    utils.DegradedDebouncer
 }
 
 var _ reconcile.Reconciler = &staticResourceReconciler{}
@@ -293,17 +294,21 @@ func (r *staticResourceReconciler) Reconcile(ctx context.Context, request reconc
 	r.logger.Debugf("reconciling after watch event %#v", request)
 	err := r.ReconcileResources(ctx)
 	if err != nil {
-		r.logger.Errorf("reconciliation failed, retrying in %s", retryInterval.String())
-		r.conditions = []configv1.ClusterOperatorStatusCondition{
-			{
-				Type:    configv1.OperatorDegraded,
-				Status:  configv1.ConditionTrue,
-				Reason:  reasonStaticResourceReconcileFailed,
-				Message: fmt.Sprintf("static resource reconciliation failed: %v", err),
-			},
+		r.degradedDebouncer.MarkDegraded()
+		r.logger.Errorf("reconciliation failed, retrying in %s (failing for %s)", retryInterval.String(), r.degradedDebouncer.DegradedFor())
+		if r.degradedDebouncer.ShouldReport() {
+			r.conditions = []configv1.ClusterOperatorStatusCondition{
+				{
+					Type:    configv1.OperatorDegraded,
+					Status:  configv1.ConditionTrue,
+					Reason:  reasonStaticResourceReconcileFailed,
+					Message: fmt.Sprintf("static resource reconciliation failed: %v", err),
+				},
+			}
 		}
 		return reconcile.Result{RequeueAfter: retryInterval}, err
 	}
+	r.degradedDebouncer.MarkNotDegraded()
 	r.conditions = []configv1.ClusterOperatorStatusCondition{}
 	return reconcile.Result{}, nil
 }

pkg/operator/utils/utils.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	"golang.org/x/mod/semver"
 	corev1 "k8s.io/api/core/v1"
@@ -492,3 +493,32 @@ func UpdateStatus(client client.Client, origCR, newCR *minterv1.CredentialsReque
 
 	return nil
 }
+
+const degradedDebounceTime = 5 * time.Minute
+
+type DegradedDebouncer struct {
+	currentlyDegraded bool
+	degradedAt        time.Time
+}
+
+func (d *DegradedDebouncer) MarkDegraded() {
+	if !d.currentlyDegraded {
+		d.currentlyDegraded = true
+		d.degradedAt = time.Now()
+	}
+}
+
+func (d *DegradedDebouncer) MarkNotDegraded() {
+	d.currentlyDegraded = false
+}
+
+func (d *DegradedDebouncer) DegradedFor() time.Duration {
+	if d.currentlyDegraded {
+		return time.Since(d.degradedAt)
+	}
+	return time.Duration(0)
+}
+
+func (d *DegradedDebouncer) ShouldReport() bool {
+	return d.DegradedFor() > degradedDebounceTime
+}