Skip to content

fix: StatefulSet stuck with 0 replicas #1902

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dcoppa wants to merge 4 commits into
base: 0.26.0
Choose a base branch
from
Open

fix: StatefulSet stuck with 0 replicas #1902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
508f843
fix: Treat StatefulSet with 0 replicas as ready to prevent reconcilia…
dcoppa Jan 9, 2026
aacca49
Merge branch 'Altinity:0.26.0' into 0.26.0
dcoppa Jan 13, 2026
61c2a17
Use a dedicated function to check for StatefulSets stuck at 0 replicas
dcoppa Jan 13, 2026
9fa2eb0
zap empty line
dcoppa Jan 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pkg/controller/chi/worker-reconciler-chi.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *api.ClickHouseInstal

new = w.buildCR(ctx, new)

// Check for StatefulSets stuck at 0 replicas when they should have more.
// This must run before ActionPlan check to handle stuck state.
common.FixStuckStatefulSets(ctx, w.a, w.c.kube.STS(), new)

switch {
case new.Spec.Suspend.Value():
// if CR is suspended, should skip reconciliation
Expand Down
4 changes: 4 additions & 0 deletions pkg/controller/chk/worker-reconciler-chk.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *apiChk.ClickHouseKee

new = w.buildCR(ctx, new)

// Check for StatefulSets stuck at 0 replicas when they should have more.
// This must run before ActionPlan check to handle stuck state.
common.FixStuckStatefulSets(ctx, w.a, w.c.kube.STS(), new)

switch {
case new.Spec.Suspend.Value():
// if CR is suspended, should skip reconciliation
Expand Down
25 changes: 25 additions & 0 deletions pkg/controller/common/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,15 @@
package common

import (
"context"

"gopkg.in/d4l3k/messagediff.v1"
apps "k8s.io/api/apps/v1"

can "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer"
log "github.com/altinity/clickhouse-operator/pkg/announcer"
api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
"github.com/altinity/clickhouse-operator/pkg/interfaces"
"github.com/altinity/clickhouse-operator/pkg/util"
)

Expand Down Expand Up @@ -68,3 +72,24 @@ func DumpStatefulSetDiff(host *api.Host, cur, new *apps.StatefulSet) {
}
}
}

// FixStuckStatefulSets checks all hosts in a CR for StatefulSets stuck at 0 replicas
// and forces them to scale up to the desired replica count.
func FixStuckStatefulSets(ctx context.Context, announcer can.Announcer, sts interfaces.IKubeSTS, cr api.ICustomResource) {
cr.WalkHosts(func(host *api.Host) error {
curStatefulSet := host.Runtime.CurStatefulSet
if curStatefulSet == nil {
return nil
}

desiredReplicas := host.GetStatefulSetReplicasNum(false)
if curStatefulSet.Spec.Replicas != nil && *curStatefulSet.Spec.Replicas == 0 && desiredReplicas != nil && *desiredReplicas > 0 {
announcer.V(1).M(host).Info("StatefulSet stuck at 0 replicas, forcing scale to %d", *desiredReplicas)
curStatefulSet.Spec.Replicas = desiredReplicas
if _, err := sts.Update(ctx, curStatefulSet); err != nil {
announcer.V(1).M(host).Error("Failed to scale up stuck StatefulSet: %v", err)
}
}
return nil
})
}