Skip to content

Commit b37644c

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #17932 from rifelpet/lbc-testexec
Migrate karpenter and LBC scenario scripts to use --test=exec
2 parents 3b75c3a + ff86c0a commit b37644c

File tree

4 files changed

+229
-134
lines changed

4 files changed

+229
-134
lines changed

‎tests/e2e/scenarios/aws-lb-controller/run-test.sh‎

Lines changed: 23 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -15,58 +15,31 @@
1515
# limitations under the License.
1616

1717
REPO_ROOT=$(git rev-parse --show-toplevel);
18-
source "${REPO_ROOT}"/tests/e2e/scenarios/lib/common.sh
18+
TEST_ROOT="${REPO_ROOT}/tests/e2e/scenarios/aws-lb-controller"
1919

20-
kops-acquire-latest
20+
make test-e2e-install
2121

22+
KUBETEST2_ARGS=()
23+
KUBETEST2_ARGS+=("-v=2")
2224

23-
# shellcheck disable=SC2034
24-
NETWORKING="amazonvpc"
25-
26-
OVERRIDES="${OVERRIDES-} --set=cluster.spec.cloudProvider.aws.loadBalancerController.enabled=true"
27-
OVERRIDES="${OVERRIDES} --set=cluster.spec.certManager.enabled=true"
28-
OVERRIDES="${OVERRIDES} --master-size=t3.medium --node-size=t3.medium" # Use amd64 because LBC's E2E suite uses single-arch amd64 test images
29-
OVERRIDES="${OVERRIDES} --image=${INSTANCE_IMAGE:-ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id}"
30-
31-
# shellcheck disable=SC2034
32-
ZONES="eu-west-1a,eu-west-1b,eu-west-1c"
33-
34-
kops-up
35-
36-
KUBECONFIG=$(mktemp -t kops.XXXXXXXXX)
37-
export KUBECONFIG
38-
"${KOPS}" export kubecfg --name "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
39-
40-
VPC=$(${KOPS} toolbox dump -o json | jq -r .vpc.id)
41-
42-
ZONE=$(${KOPS} get ig -o json | jq -r '[.[] | select(.spec.role=="Node") | .spec.subnets[0]][0]')
43-
44-
REGION=${ZONE%?}
45-
46-
REPORT_DIR="${ARTIFACTS:-$(pwd)/_artifacts}/aws-lb-controller"
47-
48-
# shellcheck disable=SC2164
49-
cd "$(mktemp -dt kops.XXXXXXXXX)"
50-
go install github.com/onsi/ginkgo/ginkgo@latest
51-
52-
LBC_VERSION=$(kubectl get deployment -n kube-system aws-load-balancer-controller -o jsonpath='{.spec.template.spec.containers[?(@.name=="controller")].image}' | cut -d':' -f2-)
53-
CLONE_ARGS=
54-
if [ -n "$LBC_VERSION" ]; then
55-
CLONE_ARGS="-b ${LBC_VERSION}"
25+
if [[ "${JOB_TYPE}" == "presubmit" && "${REPO_OWNER}/${REPO_NAME}" == "kubernetes/kops" ]]; then
26+
KUBETEST2_ARGS+=("--build")
27+
KUBETEST2_ARGS+=("--kops-binary-path=${GOPATH}/src/k8s.io/kops/.build/dist/linux/$(go env GOARCH)/kops")
28+
else
29+
KUBETEST2_ARGS+=("--kops-version-marker=${KOPS_VERSION_MARKER:-https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt}")
5630
fi
57-
# shellcheck disable=SC2086
58-
git clone ${CLONE_ARGS} https://github.com/kubernetes-sigs/aws-load-balancer-controller .
59-
60-
mkdir -p "${REPORT_DIR}"
61-
62-
ginkgo -v -r test/e2e/ingress -- \
63-
-cluster-name="${CLUSTER_NAME}" \
64-
-aws-region="${REGION}" \
65-
-aws-vpc-id="$VPC" \
66-
-ginkgo.junit-report="${REPORT_DIR}/junit-ingress.xml"
6731

68-
ginkgo -v -r test/e2e/service -- \
69-
-cluster-name="${CLUSTER_NAME}" \
70-
-aws-region="${REGION}" \
71-
-aws-vpc-id="$VPC" \
72-
-ginkgo.junit-report="${REPORT_DIR}/junit-service.xml"
32+
CREATE_ARGS="--networking amazonvpc"
33+
CREATE_ARGS="${CREATE_ARGS} --set=cluster.spec.cloudProvider.aws.loadBalancerController.enabled=true"
34+
CREATE_ARGS="${CREATE_ARGS} --set=cluster.spec.certManager.enabled=true"
35+
CREATE_ARGS="${CREATE_ARGS} --master-size=t3.medium --node-size=t3.medium"
36+
CREATE_ARGS="${CREATE_ARGS} --image=${INSTANCE_IMAGE:-ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id}"
37+
CREATE_ARGS="${CREATE_ARGS} --zones=eu-west-1a,eu-west-1b,eu-west-1c"
38+
39+
kubetest2 kops \
40+
--up --down \
41+
"${KUBETEST2_ARGS[@]}" \
42+
--cloud-provider=aws \
43+
--create-args="${CREATE_ARGS}" \
44+
--kubernetes-version="https://dl.k8s.io/release/stable.txt" \
45+
--test=exec -- "${TEST_ROOT}/test.sh"

‎tests/e2e/scenarios/aws-lb-controller/test.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2026 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
# Get the cluster name from the KUBECONFIG context
22+
CLUSTER_NAME=$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')
23+
24+
# Get VPC ID by looking at the cluster's subnet
25+
VPC=$(kubectl get nodes -o jsonpath='{.items[0].spec.providerID}' | cut -d'/' -f5 | xargs -I{} aws ec2 describe-instances --instance-ids {} --query 'Reservations[0].Instances[0].VpcId' --output text)
26+
27+
# Get the zone from a node
28+
ZONE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.topology\.kubernetes\.io/zone}')
29+
REGION=${ZONE%?}
30+
31+
REPORT_DIR="${ARTIFACTS:-$(pwd)/_artifacts}/aws-lb-controller"
32+
mkdir -p "${REPORT_DIR}"
33+
34+
# Clone the aws-load-balancer-controller repo at the version deployed in the cluster
35+
LBC_VERSION=$(kubectl get deployment -n kube-system aws-load-balancer-controller -o jsonpath='{.spec.template.spec.containers[?(@.name=="controller")].image}' | cut -d':' -f2-)
36+
TEMPDIR=$(mktemp -dt kops.XXXXXXXXX)
37+
cd "${TEMPDIR}"
38+
39+
go install github.com/onsi/ginkgo/v2/ginkgo@latest
40+
41+
CLONE_ARGS=
42+
if [ -n "$LBC_VERSION" ]; then
43+
CLONE_ARGS="-b ${LBC_VERSION}"
44+
fi
45+
# shellcheck disable=SC2086
46+
git clone ${CLONE_ARGS} https://github.com/kubernetes-sigs/aws-load-balancer-controller .
47+
48+
ginkgo -v -r test/e2e/ingress -- \
49+
-cluster-name="${CLUSTER_NAME}" \
50+
-aws-region="${REGION}" \
51+
-aws-vpc-id="$VPC" \
52+
-ginkgo.junit-report="${REPORT_DIR}/junit-ingress.xml"
53+
54+
ginkgo -v -r test/e2e/service -- \
55+
-cluster-name="${CLUSTER_NAME}" \
56+
-aws-region="${REGION}" \
57+
-aws-vpc-id="$VPC" \
58+
-ginkgo.junit-report="${REPORT_DIR}/junit-service.xml"

‎tests/e2e/scenarios/karpenter/run-test.sh‎

Lines changed: 29 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -15,87 +15,32 @@
1515
# limitations under the License.
1616

1717
REPO_ROOT=$(git rev-parse --show-toplevel);
18-
source "${REPO_ROOT}"/tests/e2e/scenarios/lib/common.sh
19-
20-
kops-acquire-latest
21-
22-
NETWORKING=cilium
23-
OVERRIDES="${OVERRIDES-} --instance-manager=karpenter"
24-
OVERRIDES="${OVERRIDES} --control-plane-size=c6g.large"
25-
OVERRIDES="${OVERRIDES} --set=cluster.spec.karpenter.featureGates=StaticCapacity=true"
26-
27-
kops-up
28-
29-
# Get the node instance group user data script from the kOps state store
30-
USER_DATA=$(aws s3 cp "${KOPS_STATE_STORE-}/${CLUSTER_NAME}/igconfig/node/nodes/nodeupscript.sh" -)
31-
# Indent the user data script for embedding in the EC2NodeClass
32-
USER_DATA=${USER_DATA//$'\n'/$'\n '}
33-
34-
# Create a EC2NodeClass for Karpenter
35-
kubectl apply -f - <<YAML
36-
apiVersion: karpenter.k8s.aws/v1
37-
kind: EC2NodeClass
38-
metadata:
39-
name: default
40-
spec:
41-
amiFamily: Custom
42-
amiSelectorTerms:
43-
- ssmParameter: /aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id
44-
associatePublicIPAddress: true
45-
tags:
46-
KubernetesCluster: ${CLUSTER_NAME}
47-
kops.k8s.io/instancegroup: nodes
48-
k8s.io/role/node: "1"
49-
subnetSelectorTerms:
50-
- tags:
51-
KubernetesCluster: ${CLUSTER_NAME}
52-
securityGroupSelectorTerms:
53-
- tags:
54-
KubernetesCluster: ${CLUSTER_NAME}
55-
Name: nodes.${CLUSTER_NAME}
56-
instanceProfile: nodes.${CLUSTER_NAME}
57-
userData: |
58-
${USER_DATA}
59-
YAML
60-
61-
# Create a NodePool for Karpenter
62-
# Effectively disable consolidation for 30 minutes to avoid flakes in the tests
63-
kubectl apply -f - <<YAML
64-
apiVersion: karpenter.sh/v1
65-
kind: NodePool
66-
metadata:
67-
name: default
68-
spec:
69-
template:
70-
spec:
71-
requirements:
72-
- key: node.kubernetes.io/instance-type
73-
operator: In
74-
values: ["m6g.large"]
75-
- key: karpenter.sh/capacity-type
76-
operator: In
77-
values: ["on-demand"]
78-
nodeClassRef:
79-
group: karpenter.k8s.aws
80-
kind: EC2NodeClass
81-
name: default
82-
replicas: 4
83-
disruption:
84-
consolidationPolicy: WhenEmpty
85-
consolidateAfter: 30m
86-
YAML
87-
88-
# Wait for the nodes to start being provisioned
89-
sleep 30
90-
# Wait for the nodes to be ready
91-
"${KOPS}" validate cluster --wait=10m
92-
93-
# Run the tests
94-
cp "${KOPS}" "${WORKSPACE}/kops"
95-
${KUBETEST2} \
96-
--test=kops \
97-
--kops-binary-path="${KOPS}" \
98-
-- \
99-
--test-package-version="${K8S_VERSION}" \
100-
--focus-regex="\[Conformance\]" \
101-
--parallel 20
18+
TEST_ROOT="${REPO_ROOT}/tests/e2e/scenarios/karpenter"
19+
20+
make test-e2e-install
21+
22+
KUBETEST2_ARGS=()
23+
KUBETEST2_ARGS+=("-v=2")
24+
25+
if [[ "${JOB_TYPE}" == "presubmit" && "${REPO_OWNER}/${REPO_NAME}" == "kubernetes/kops" ]]; then
26+
KUBETEST2_ARGS+=("--build")
27+
KUBETEST2_ARGS+=("--kops-binary-path=${GOPATH}/src/k8s.io/kops/.build/dist/linux/$(go env GOARCH)/kops")
28+
else
29+
KUBETEST2_ARGS+=("--kops-version-marker=${KOPS_VERSION_MARKER:-https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt}")
30+
fi
31+
32+
CREATE_ARGS="--networking cilium"
33+
CREATE_ARGS="${CREATE_ARGS} --instance-manager=karpenter"
34+
CREATE_ARGS="${CREATE_ARGS} --control-plane-size=c6g.large"
35+
CREATE_ARGS="${CREATE_ARGS} --set=cluster.spec.karpenter.featureGates=StaticCapacity=true"
36+
37+
K8S_VERSION="${K8S_VERSION:-$(curl -s -L https://dl.k8s.io/release/stable.txt)}"
38+
39+
kubetest2 kops \
40+
--up --down \
41+
"${KUBETEST2_ARGS[@]}" \
42+
--cloud-provider=aws \
43+
--create-args="${CREATE_ARGS}" \
44+
--kubernetes-version="${K8S_VERSION}" \
45+
--env=K8S_VERSION="${K8S_VERSION}" \
46+
--test=exec -- "${TEST_ROOT}/test.sh"

‎tests/e2e/scenarios/karpenter/test.sh‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2026 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -o errexit
18+
set -o nounset
19+
set -o pipefail
20+
21+
# Get the cluster name from the KUBECONFIG context
22+
CLUSTER_NAME=$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')
23+
24+
# Get KOPS_STATE_STORE from kubeconfig server URL (the bucket name is embedded)
25+
# Alternatively, we can get it from environment if kubetest2 passes it
26+
if [[ -z "${KOPS_STATE_STORE:-}" ]]; then
27+
echo "KOPS_STATE_STORE must be set"
28+
exit 1
29+
fi
30+
31+
# Get the node instance group user data script from the kOps state store
32+
USER_DATA=$(aws s3 cp "${KOPS_STATE_STORE}/${CLUSTER_NAME}/igconfig/node/nodes/nodeupscript.sh" -)
33+
# Indent the user data script for embedding in the EC2NodeClass
34+
USER_DATA=${USER_DATA//$'\n'/$'\n '}
35+
36+
# Create a EC2NodeClass for Karpenter
37+
kubectl apply -f - <<YAML
38+
apiVersion: karpenter.k8s.aws/v1
39+
kind: EC2NodeClass
40+
metadata:
41+
name: default
42+
spec:
43+
amiFamily: Custom
44+
amiSelectorTerms:
45+
- ssmParameter: /aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id
46+
associatePublicIPAddress: true
47+
tags:
48+
KubernetesCluster: ${CLUSTER_NAME}
49+
kops.k8s.io/instancegroup: nodes
50+
k8s.io/role/node: "1"
51+
subnetSelectorTerms:
52+
- tags:
53+
KubernetesCluster: ${CLUSTER_NAME}
54+
securityGroupSelectorTerms:
55+
- tags:
56+
KubernetesCluster: ${CLUSTER_NAME}
57+
Name: nodes.${CLUSTER_NAME}
58+
instanceProfile: nodes.${CLUSTER_NAME}
59+
userData: |
60+
${USER_DATA}
61+
YAML
62+
63+
# Create a NodePool for Karpenter
64+
# Effectively disable consolidation for 30 minutes to avoid flakes in the tests
65+
kubectl apply -f - <<YAML
66+
apiVersion: karpenter.sh/v1
67+
kind: NodePool
68+
metadata:
69+
name: default
70+
spec:
71+
template:
72+
spec:
73+
requirements:
74+
- key: node.kubernetes.io/instance-type
75+
operator: In
76+
values: ["m6g.large"]
77+
- key: karpenter.sh/capacity-type
78+
operator: In
79+
values: ["on-demand"]
80+
nodeClassRef:
81+
group: karpenter.k8s.aws
82+
kind: EC2NodeClass
83+
name: default
84+
replicas: 4
85+
disruption:
86+
consolidationPolicy: WhenEmpty
87+
consolidateAfter: 30m
88+
YAML
89+
90+
# Wait for the nodes to start being provisioned
91+
sleep 30
92+
93+
# Download kops to validate cluster
94+
KOPS_BASE_URL="$(curl -s https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci-updown-green.txt)"
95+
KOPS=$(mktemp -t kops.XXXXXXXXX)
96+
wget -qO "${KOPS}" "$KOPS_BASE_URL/$(go env GOOS)/$(go env GOARCH)/kops"
97+
chmod +x "${KOPS}"
98+
99+
# Wait for the nodes to be ready
100+
"${KOPS}" validate cluster --wait=10m
101+
102+
if [[ -z "${K8S_VERSION:-}" ]]; then
103+
K8S_VERSION="$(curl -s -L https://dl.k8s.io/release/stable.txt)"
104+
fi
105+
106+
# Download test binaries
107+
BINDIR=$(mktemp -d)
108+
wget -qO- "https://dl.k8s.io/${K8S_VERSION}/kubernetes-test-linux-amd64.tar.gz" | tar xz -C "${BINDIR}" --strip-components=3 kubernetes/test/bin/e2e.test kubernetes/test/bin/ginkgo
109+
110+
# Run conformance tests
111+
"${BINDIR}/ginkgo" \
112+
--nodes=20 \
113+
--focus="\[Conformance\]" \
114+
--no-color \
115+
"${BINDIR}/e2e.test" \
116+
-- \
117+
--provider=skeleton \
118+
--kubeconfig="${KUBECONFIG:-${HOME}/.kube/config}" \
119+
--report-dir="${ARTIFACTS:-/tmp}"

0 commit comments

Comments
 (0)