Skip to content

Commit ecd375b

Browse files
SylvainSenechalSylvainSenechal
committed
update fix zookeeper
1 parent 7f89474 commit ecd375b

File tree

5 files changed

+129
-25
lines changed

5 files changed

+129
-25
lines changed

‎.devcontainer/setup.sh‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,19 @@
33
set -e
44

55
env_variables=$(yq eval '.env | to_entries | .[] | .key + "=" + .value' .github/workflows/end2end.yaml | sed 's/\${{[^}]*}}//g') && export $env_variables
6-
export GIT_ACCESS_TOKEN=${GITHUB_TOKEN}
6+
7+
# In CI, GIT_ACCESS_TOKEN comes from a GitHub App token.
8+
# Locally, we need the user to provide one (usually via GITHUB_TOKEN).
9+
if [[ -z "${GIT_ACCESS_TOKEN:-}" ]]; then
10+
export GIT_ACCESS_TOKEN="${GITHUB_TOKEN:-}"
11+
fi
12+
13+
if [[ -z "${GIT_ACCESS_TOKEN:-}" ]]; then
14+
echo "ERROR: Missing GitHub token. Set GITHUB_TOKEN (or GIT_ACCESS_TOKEN) with access to scality/zenko-operator." >&2
15+
echo "Example: export GITHUB_TOKEN=ghp_***" >&2
16+
exit 1
17+
fi
18+
719
export E2E_IMAGE_TAG=latest
820

921
# Disable GCP tests as we don't have credentials setup in devcontainer
@@ -21,11 +33,27 @@ for i in $(seq 0 $array_length); do
2133
#step=$(yq ".runs.steps[$i]" .github/actions/deploy/action.yaml)
2234
working_dir=$(yq ".runs.steps[$i].working-directory" .github/actions/deploy/action.yaml)
2335
run_command=$(yq ".runs.steps[$i].run" .github/actions/deploy/action.yaml)
36+
step_if=$(yq ".runs.steps[$i].if" .github/actions/deploy/action.yaml)
2437

2538
# We don't want to run `run-e2e-test.sh` because it is used for linting here, user will run it manually if needed after deployment
2639
# We can't run `configure-e2e.sh` here because it needs an image that is not yet built and sent to kind, will be run after
2740
(
28-
if [[ "$run_command" != "null" && "$run_command" != *"configure-e2e.sh"* && "$run_command" != *"run-e2e-test.sh"* ]]; then
41+
should_run=true
42+
43+
# Best-effort support for composite action `if:` (CI evaluates these, local runner must emulate).
44+
if [[ "$step_if" != "null" ]]; then
45+
# Only conditional step in the deploy action today.
46+
if [[ "$step_if" == *"inputs.deploy_metadata"* ]]; then
47+
if [[ "${GITHUB_INPUTS_deploy_metadata:-false}" != "true" ]]; then
48+
should_run=false
49+
fi
50+
else
51+
echo "Skipping step with unsupported condition: $step_if"
52+
should_run=false
53+
fi
54+
fi
55+
56+
if [[ "$should_run" == "true" && "$run_command" != "null" && "$run_command" != *"configure-e2e.sh"* && "$run_command" != *"run-e2e-test.sh"* ]]; then
2957
# Inject env 'generated' from previous steps
3058
source "$GITHUB_ENV"
3159

‎.github/scripts/end2end/configure-e2e-ctst.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ set -exu
33

44
# Setup test environment variables
55
export ZENKO_NAME=${1:-"end2end"}
6+
export NAMESPACE=${NAMESPACE:-default}
7+
echo "=== Running configure-e2e-ctst.sh (ZENKO_NAME=${ZENKO_NAME}, NAMESPACE=${NAMESPACE}) ==="
8+
69
# Getting kafka host from backbeat's config
710
KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
811
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)

‎.github/scripts/end2end/configure-e2e.sh‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ ZENKO_NAME=${1:-end2end}
88
E2E_IMAGE=${2:-ghcr.io/scality/zenko/zenko-e2e:latest}
99
NAMESPACE=${3:-default}
1010

11+
echo "=== Running configure-e2e.sh (ZENKO_NAME=${ZENKO_NAME}, E2E_IMAGE=${E2E_IMAGE}, NAMESPACE=${NAMESPACE}) ==="
12+
1113
SERVICE_ACCOUNT="${ZENKO_NAME}-config"
1214
POD_NAME="${ZENKO_NAME}-config"
1315
MANAGEMENT_ENDPOINT="http://${ZENKO_NAME}-management-orbit-api:5001"
@@ -52,10 +54,33 @@ KAFKA_REGISTRY_NAME=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yam
5254
KAFKA_IMAGE_NAME=$(yq eval ".kafka.image" ../../../solution/deps.yaml)
5355
KAFKA_IMAGE_TAG=$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
5456
KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG
55-
KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
56-
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
57+
58+
BACKBEAT_CONFIG_SELECTOR="app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=${ZENKO_NAME}"
59+
60+
# backbeat-config is produced by the operator during reconciliation; wait for it.
61+
for i in $(seq 1 120); do
62+
SECRET_NAME=$(kubectl -n ${NAMESPACE} get secret -l "${BACKBEAT_CONFIG_SELECTOR}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
63+
if [ -n "${SECRET_NAME}" ]; then
64+
break
65+
fi
66+
sleep 5
67+
done
68+
69+
if [ -z "${SECRET_NAME:-}" ]; then
70+
echo "Timed out waiting for backbeat-config secret (selector: ${BACKBEAT_CONFIG_SELECTOR}) in namespace ${NAMESPACE}" >&2
71+
kubectl -n ${NAMESPACE} get secret -l "${BACKBEAT_CONFIG_SELECTOR}" -o yaml || true
72+
exit 1
73+
fi
74+
75+
KAFKA_HOST_PORT=$(kubectl -n ${NAMESPACE} get secret -l "${BACKBEAT_CONFIG_SELECTOR}" \
76+
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
5777
KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
5878

79+
if [ -z "${KAFKA_HOST_PORT}" ]; then
80+
echo "Kafka bootstrap address is empty (from secret ${SECRET_NAME} in namespace ${NAMESPACE})" >&2
81+
exit 1
82+
fi
83+
5984
# Creating replication/transition and notification topics in kafka
6085
kubectl run kafka-topics \
6186
--image=$KAFKA_IMAGE \

‎.github/scripts/end2end/fix-zookeeper.sh‎

Lines changed: 66 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ ZK_STS_NAME="${ZENKO_NAME}-base-quorum"
1212
ZK_CONTAINER_NAME="zookeeper"
1313
ZK_POD_NAME="${ZK_STS_NAME}-0"
1414

15+
ZK_JVMFLAGS="-Xmx512m -Xms512m -XX:-UseContainerSupport -XX:ActiveProcessorCount=1 -Djava.awt.headless=true -Dzookeeper.log.dir=/data/logs -Dzookeeper.root.logger=INFO,CONSOLE -Dlog4j.configuration=file:/data/conf/log4j.properties"
16+
1517
OPERATOR_WAIT_TIMEOUT=120
1618
STATEFULSET_WAIT_TIMEOUT=180
1719

@@ -61,37 +63,83 @@ while true; do
6163
sleep 2
6264
done
6365

64-
# Patch the StatefulSet with JVM flags to disable container support
65-
# as ubuntu runners now are incompatible with zookeeper.
66-
kubectl -n "${NAMESPACE}" patch statefulset "${ZK_STS_NAME}" --type='strategic' \
66+
# IMPORTANT:
67+
# The zookeeper-operator reconciles the StatefulSet from the ZookeeperCluster CR.
68+
# Patching the StatefulSet directly is often reverted immediately.
69+
# Patch the ZookeeperCluster spec instead so it persists (CI + lab).
70+
echo "Waiting for ZookeeperCluster (${ZK_STS_NAME}) to be created by operator..."
71+
ZKCLUSTER_WAIT_START=$(date +%s)
72+
while true; do
73+
ELAPSED=$(get_elapsed ${ZKCLUSTER_WAIT_START})
74+
if [ ${ELAPSED} -ge ${STATEFULSET_WAIT_TIMEOUT} ]; then
75+
echo "ERROR: Timed out after ${STATEFULSET_WAIT_TIMEOUT}s waiting for ZookeeperCluster ${ZK_STS_NAME}."
76+
kubectl -n "${NAMESPACE}" get zookeeperclusters.zookeeper.pravega.io -o wide 2>/dev/null || true
77+
exit 1
78+
fi
79+
if kubectl -n "${NAMESPACE}" get zookeepercluster "${ZK_STS_NAME}" > /dev/null 2>&1; then
80+
break
81+
fi
82+
sleep 2
83+
done
84+
85+
echo "Patching ZookeeperCluster with JVMFLAGS workaround..."
86+
kubectl -n "${NAMESPACE}" patch zookeepercluster "${ZK_STS_NAME}" --type='merge' \
6787
-p '{
6888
"spec": {
69-
"template": {
70-
"spec": {
71-
"containers": [
89+
"containers": [
90+
{
91+
"name": "'"${ZK_CONTAINER_NAME}"'",
92+
"env": [
7293
{
73-
"name": "'"${ZK_CONTAINER_NAME}"'",
74-
"env": [
75-
{
76-
"name": "JVMFLAGS",
77-
"value": "-Xmx512m -Xms512m -XX:-UseContainerSupport -XX:ActiveProcessorCount=1 -Djava.awt.headless=true -Dzookeeper.log.dir=/data/logs -Dzookeeper.root.logger=INFO,CONSOLE -Dlog4j.configuration=file:/data/conf/log4j.properties"
78-
}
79-
]
94+
"name": "JVMFLAGS",
95+
"value": "'"${ZK_JVMFLAGS}"'"
8096
}
8197
]
8298
}
83-
}
99+
]
84100
}
85101
}'
86102

103+
echo "Verifying JVMFLAGS is present on ZookeeperCluster spec..."
104+
if ! kubectl -n "${NAMESPACE}" get zookeepercluster "${ZK_STS_NAME}" \
105+
-o jsonpath='{.spec.containers[?(@.name=="'"${ZK_CONTAINER_NAME}"'")].env[?(@.name=="JVMFLAGS")].value}{"\n"}' \
106+
| grep -q -- "-XX:-UseContainerSupport"; then
107+
echo "ERROR: JVMFLAGS not present on ZookeeperCluster spec after patch." >&2
108+
kubectl -n "${NAMESPACE}" get zookeepercluster "${ZK_STS_NAME}" -o yaml | sed -n '1,220p' >&2 || true
109+
exit 1
110+
fi
111+
87112

88113
# Delete the pod to apply the patch
89114
kubectl delete pod "${ZK_POD_NAME}" -n "${NAMESPACE}" --ignore-not-found=true --wait=false
90115

91-
# Wait for the pod to become Ready
92-
if ! kubectl wait --for=condition=Ready "pod/${ZK_POD_NAME}" --timeout=300s -n "${NAMESPACE}"; then
93-
echo "ERROR: Zookeeper pod ${ZK_POD_NAME} failed to become Ready after patching."
116+
# Wait for the StatefulSet to recreate the pod, then for it to become Ready.
117+
POD_WAIT_TIMEOUT=300
118+
POD_WAIT_START=$(date +%s)
119+
120+
while true; do
121+
ELAPSED=$(get_elapsed ${POD_WAIT_START})
122+
if [ ${ELAPSED} -ge ${POD_WAIT_TIMEOUT} ]; then
123+
echo "ERROR: Timed out after ${POD_WAIT_TIMEOUT}s waiting for ${ZK_POD_NAME} to be recreated and become Ready." >&2
124+
echo "--- StatefulSet status ---" >&2
125+
kubectl -n "${NAMESPACE}" get sts "${ZK_STS_NAME}" -o wide >&2 || true
126+
kubectl -n "${NAMESPACE}" describe sts "${ZK_STS_NAME}" | sed -n '1,200p' >&2 || true
127+
echo "--- Pod list (matching quorum) ---" >&2
128+
kubectl -n "${NAMESPACE}" get pods -o wide | grep "${ZK_STS_NAME}" >&2 || true
129+
echo "--- Zookeeper pod describe ---" >&2
130+
kubectl -n "${NAMESPACE}" describe pod "${ZK_POD_NAME}" >&2 || true
131+
echo "--- Zookeeper pod logs (tail) ---" >&2
132+
kubectl -n "${NAMESPACE}" logs "${ZK_POD_NAME}" --tail=120 >&2 || true
94133
exit 1
95-
fi
134+
fi
135+
136+
if kubectl -n "${NAMESPACE}" get pod "${ZK_POD_NAME}" > /dev/null 2>&1; then
137+
if kubectl -n "${NAMESPACE}" wait --for=condition=Ready "pod/${ZK_POD_NAME}" --timeout=10s > /dev/null 2>&1; then
138+
break
139+
fi
140+
fi
141+
142+
sleep 2
143+
done
96144

97145
echo "Zookeeper fix applied successfully."

‎.github/scripts/end2end/install-kind-dependencies.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,11 @@ helm repo add --force-update banzaicloud-stable https://kubernetes-charts.banzai
5151
echo -n "::notice file=$(basename $0),line=$LINENO,title=Banzaicloud Charts not available::"
5252
echo "Failed to add banzaicloud-stable repo, using local checkout"
5353

54-
kafa_operator="$(mktemp -d)"
54+
kafka_operator="$(mktemp -d)"
5555
git -c advice.detachedHead=false clone -q --depth 1 -b "v${KAFKA_OPERATOR_VERSION}" \
56-
https://github.com/banzaicloud/koperator "${kafa_operator}"
56+
https://github.com/banzaicloud/koperator "${kafka_operator}"
5757

58-
KAFKA_CHART="${kafa_operator}/charts/kafka-operator"
58+
KAFKA_CHART="${kafka_operator}/charts/kafka-operator"
5959
}
6060
helm repo update
6161

0 commit comments

Comments
 (0)