diff --git a/.gitignore b/.gitignore index 7a7feec..848052c 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ go.work *.swp *.swo *~ + +local-dev \ No newline at end of file diff --git a/Makefile b/Makefile index 2805c02..84e6ea5 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,110 @@ else GOBIN=$(shell go env GOBIN) endif -KIND_CLUSTER ?= kind +KIND_CLUSTER ?= dbaas-controller +KIND_NETWORK ?= dbaas-controller + +KIND_VERSION = v0.25.0 +KUBECTL_VERSION := v1.31.0 +HELM_VERSION := v3.16.1 +GOJQ_VERSION = v0.12.16 +KUSTOMIZE_VERSION := v5.4.3 + +HELM = $(realpath ./local-dev/helm) +KUBECTL = $(realpath ./local-dev/kubectl) +JQ = $(realpath ./local-dev/jq) +KIND = $(realpath ./local-dev/kind) +KUSTOMIZE = $(realpath ./local-dev/kustomize) + +ARCH := $(shell uname | tr '[:upper:]' '[:lower:]') + +.PHONY: local-dev/kind +local-dev/kind: +ifeq ($(KIND_VERSION), $(shell kind version 2>/dev/null | sed -nE 's/kind (v[0-9.]+).*/\1/p')) + $(info linking local kind version $(KIND_VERSION)) + ln -sf $(shell command -v kind) ./local-dev/kind +else +ifneq ($(KIND_VERSION), $(shell ./local-dev/kind version 2>/dev/null | sed -nE 's/kind (v[0-9.]+).*/\1/p')) + $(info downloading kind version $(KIND_VERSION) for $(ARCH)) + mkdir -p local-dev + rm local-dev/kind || true + curl -sSLo local-dev/kind https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$(ARCH)-amd64 + chmod a+x local-dev/kind +endif +endif + +.PHONY: local-dev/kustomize +local-dev/kustomize: +ifeq ($(KUSTOMIZE_VERSION), $(shell kustomize version 2>/dev/null | sed -nE 's/(v[0-9.]+).*/\1/p')) + $(info linking local kustomize version $(KUSTOMIZE_VERSION)) + ln -sf $(shell command -v kustomize) ./local-dev/kustomize +else +ifneq ($(KUSTOMIZE_VERSION), $(shell ./local-dev/kustomize version 2>/dev/null | sed -nE 's/(v[0-9.]+).*/\1/p')) + $(info downloading kustomize version $(KUSTOMIZE_VERSION) for $(ARCH)) + rm local-dev/kustomize || true + curl -sSL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F$(KUSTOMIZE_VERSION)/kustomize_$(KUSTOMIZE_VERSION)_$(ARCH)_amd64.tar.gz | tar -xzC local-dev + chmod a+x local-dev/kustomize +endif +endif + +.PHONY: local-dev/helm +local-dev/helm: +ifeq ($(HELM_VERSION), $(shell helm version --short --client 2>/dev/null | sed -nE 's/(v[0-9.]+).*/\1/p')) + $(info linking local helm version $(HELM_VERSION)) + ln -sf $(shell command -v helm) ./local-dev/helm +else +ifneq ($(HELM_VERSION), $(shell ./local-dev/helm version --short --client 2>/dev/null | sed -nE 's/(v[0-9.]+).*/\1/p')) + $(info downloading helm version $(HELM_VERSION) for $(ARCH)) + rm local-dev/helm || true + curl -sSL https://get.helm.sh/helm-$(HELM_VERSION)-$(ARCH)-amd64.tar.gz | tar -xzC local-dev --strip-components=1 $(ARCH)-amd64/helm + chmod a+x local-dev/helm +endif +endif + +.PHONY: local-dev/jq +local-dev/jq: +ifeq ($(GOJQ_VERSION), $(shell gojq -v 2>/dev/null | sed -nE 's/gojq ([0-9.]+).*/v\1/p')) + $(info linking local gojq version $(GOJQ_VERSION)) + ln -sf $(shell command -v gojq) ./local-dev/jq +else +ifneq ($(GOJQ_VERSION), $(shell ./local-dev/jq -v 2>/dev/null | sed -nE 's/gojq ([0-9.]+).*/v\1/p')) + $(info downloading gojq version $(GOJQ_VERSION) for $(ARCH)) + rm local-dev/jq || true +ifeq ($(ARCH), darwin) + TMPDIR=$$(mktemp -d) \ + && curl -sSL https://github.com/itchyny/gojq/releases/download/$(GOJQ_VERSION)/gojq_$(GOJQ_VERSION)_$(ARCH)_arm64.zip -o $$TMPDIR/gojq.zip \ + && (cd $$TMPDIR && unzip gojq.zip) && cp $$TMPDIR/gojq_$(GOJQ_VERSION)_$(ARCH)_arm64/gojq ./local-dev/jq && rm -rf $$TMPDIR +else + curl -sSL https://github.com/itchyny/gojq/releases/download/$(GOJQ_VERSION)/gojq_$(GOJQ_VERSION)_$(ARCH)_amd64.tar.gz | tar -xzC local-dev --strip-components=1 gojq_$(GOJQ_VERSION)_$(ARCH)_amd64/gojq + mv ./local-dev/{go,}jq +endif + chmod a+x local-dev/jq +endif +endif + +.PHONY: local-dev/kubectl +local-dev/kubectl: +ifeq ($(KUBECTL_VERSION), $(shell kubectl version --client 2>/dev/null | grep Client | sed -E 's/Client Version: (v[0-9.]+).*/\1/')) + $(info linking local kubectl version $(KUBECTL_VERSION)) + ln -sf $(shell command -v kubectl) ./local-dev/kubectl +else +ifneq ($(KUBECTL_VERSION), $(shell ./local-dev/kubectl version --client 2>/dev/null | grep Client | sed -E 's/Client Version: (v[0-9.]+).*/\1/')) + $(info downloading kubectl version $(KUBECTL_VERSION) for $(ARCH)) + rm local-dev/kubectl || true + curl -sSLo local-dev/kubectl https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/$(ARCH)/amd64/kubectl + chmod a+x local-dev/kubectl +endif +endif + +.PHONY: local-dev/tools +local-dev/tools: local-dev/kind local-dev/kustomize local-dev/kubectl local-dev/jq local-dev/helm + +.PHONY: helm/repos +helm/repos: local-dev/helm + # install repo dependencies required by the charts +# $(HELM) repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +# $(HELM) repo add metallb https://metallb.github.io/metallb +# $(HELM) repo update # CONTAINER_TOOL defines the container tool to be used for building images. # Be aware that the target commands are only tested with Docker which is @@ -67,29 +170,52 @@ test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out .PHONY: create-kind-cluster -create-kind-cluster: - @if ! kind get clusters | grep -q $(KIND_CLUSTER); then \ - docker network inspect $(KIND_CLUSTER) >/dev/null 2>&1 || docker network create $(KIND_CLUSTER); \ - kind create cluster --wait=60s --name=$(KIND_CLUSTER) --config=kind-config.yaml; \ - else \ - echo "Cluster $(KIND_CLUSTER) already exists"; \ - fi +create-kind-cluster: local-dev/tools helm/repos + docker network inspect $(KIND_NETWORK) >/dev/null || docker network create $(KIND_NETWORK) \ + && LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_NETWORK) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') \ + && export KIND_NODE_IP=$$(echo $${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $$1,$$2,$$3,240}' OFS='.') \ + && export KIND_EXPERIMENTAL_DOCKER_NETWORK=$(KIND_NETWORK) \ + && $(KIND) create cluster --wait=60s --name=$(KIND_CLUSTER) --config=test-resources/test-suite.kind-config.yaml + +# Create a kind cluster locally and run the test e2e test suite against it +.PHONY: kind/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up locally +kind/test-e2e: create-kind-cluster kind/re-test-e2e -.PHONY: delete-kind-cluster -delete-kind-cluster: - kind delete cluster --name=$(KIND_CLUSTER) && docker network rm $(KIND_CLUSTER) +.PHONY: local-kind/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up locally +kind/re-test-e2e: + export KIND_PATH=$(KIND) && \ + export KUBECTL_PATH=$(KUBECTL) && \ + export KIND_CLUSTER=$(KIND_CLUSTER) && \ + LAGOON_KIND_CIDR_BLOCK=$$(docker network inspect $(KIND_NETWORK) | $(JQ) '. [0].IPAM.Config[0].Subnet' | tr -d '"') && \ + export KIND_NODE_IP=$$(echo $${LAGOON_KIND_CIDR_BLOCK%???} | awk -F'.' '{print $$1,$$2,$$3,240}' OFS='.') && \ + $(KIND) export kubeconfig --name=$(KIND_CLUSTER) && \ + $(MAKE) test-e2e + +.PHONY: clean +kind/clean: + $(KIND) delete cluster --name=$(KIND_CLUSTER) && docker network rm $(KIND_NETWORK) # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. -.PHONY: github/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up inside github action. -github/test-e2e: +.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up inside github action. +test-e2e: go test ./test/e2e/ -v -ginkgo.v -# Create a kind cluster locally and run the test e2e test suite against it -.PHONY: local-kind/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up locally -local-kind/test-e2e: create-kind-cluster +# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. +.PHONY: github/test-e2e # Run the e2e tests against a Kind k8s instance that is spun up inside github action. +github/test-e2e: local-dev/tools test-e2e + +.PHONY: kind/set-kubeconfig +kind/set-kubeconfig: export KIND_CLUSTER=$(KIND_CLUSTER) && \ - kind export kubeconfig --name=$(KIND_CLUSTER) && \ - go test ./test/e2e/ -v -ginkgo.v + $(KIND) export kubeconfig --name=$(KIND_CLUSTER) + +.PHONY: kind/logs-dbaas-controller +kind/logs-dbaas-controller: + export KIND_CLUSTER=$(KIND_CLUSTER) && \ + $(KIND) export kubeconfig --name=$(KIND_CLUSTER) && \ + $(KUBECTL) -n dbaas-controller-system logs -f \ + $$($(KUBECTL) -n dbaas-controller-system get pod -l control-plane=controller-manager -o jsonpath="{.items[0].metadata.name}") \ + -c manager .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint @@ -138,7 +264,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform rm Dockerfile.cross .PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. +build-installer: manifests generate local-dev/kustomize ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist @if [ -d "config/crd" ]; then \ $(KUSTOMIZE) build config/crd > dist/install.yaml; \ @@ -154,20 +280,20 @@ ifndef ignore-not-found endif .PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. +install: manifests local-dev/kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - .PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. +uninstall: manifests local-dev/kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. +deploy: manifests local-dev/kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - .PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. +undeploy: local-dev/kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - ##@ Dependencies @@ -178,23 +304,15 @@ $(LOCALBIN): mkdir -p $(LOCALBIN) ## Tool Binaries -KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) ## Tool Versions -KUSTOMIZE_VERSION ?= v5.3.0 -CONTROLLER_TOOLS_VERSION ?= v0.16.0 +CONTROLLER_TOOLS_VERSION ?= v0.16.5 ENVTEST_VERSION ?= latest GOLANGCI_LINT_VERSION ?= v1.54.2 -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) - .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. $(CONTROLLER_GEN): $(LOCALBIN) diff --git a/config/crd/bases/crd.lagoon.sh_databaserequests.yaml b/config/crd/bases/crd.lagoon.sh_databaserequests.yaml index 9c6226b..69da86b 100644 --- a/config/crd/bases/crd.lagoon.sh_databaserequests.yaml +++ b/config/crd/bases/crd.lagoon.sh_databaserequests.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.0 + controller-gen.kubebuilder.io/version: v0.16.5 name: databaserequests.crd.lagoon.sh spec: group: crd.lagoon.sh diff --git a/config/crd/bases/crd.lagoon.sh_mongodbproviders.yaml b/config/crd/bases/crd.lagoon.sh_mongodbproviders.yaml index 3d153e7..698fd57 100644 --- a/config/crd/bases/crd.lagoon.sh_mongodbproviders.yaml +++ b/config/crd/bases/crd.lagoon.sh_mongodbproviders.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.0 + controller-gen.kubebuilder.io/version: v0.16.5 name: mongodbproviders.crd.lagoon.sh spec: group: crd.lagoon.sh diff --git a/config/crd/bases/crd.lagoon.sh_relationaldatabaseproviders.yaml b/config/crd/bases/crd.lagoon.sh_relationaldatabaseproviders.yaml index b0e5da2..df8370d 100644 --- a/config/crd/bases/crd.lagoon.sh_relationaldatabaseproviders.yaml +++ b/config/crd/bases/crd.lagoon.sh_relationaldatabaseproviders.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.0 + controller-gen.kubebuilder.io/version: v0.16.5 name: relationaldatabaseproviders.crd.lagoon.sh spec: group: crd.lagoon.sh diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8f0fe4c..c99b166 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -15,17 +15,6 @@ rules: - "" resources: - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - services verbs: - create @@ -39,57 +28,7 @@ rules: - crd.lagoon.sh resources: - databaserequests - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - crd.lagoon.sh - resources: - - databaserequests/finalizers - verbs: - - update -- apiGroups: - - crd.lagoon.sh - resources: - - databaserequests/status - verbs: - - get - - patch - - update -- apiGroups: - - crd.lagoon.sh - resources: - mongodbproviders - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - crd.lagoon.sh - resources: - - mongodbproviders/finalizers - verbs: - - update -- apiGroups: - - crd.lagoon.sh - resources: - - mongodbproviders/status - verbs: - - get - - patch - - update -- apiGroups: - - crd.lagoon.sh - resources: - relationaldatabaseproviders verbs: - create @@ -102,12 +41,16 @@ rules: - apiGroups: - crd.lagoon.sh resources: + - databaserequests/finalizers + - mongodbproviders/finalizers - relationaldatabaseproviders/finalizers verbs: - update - apiGroups: - crd.lagoon.sh resources: + - databaserequests/status + - mongodbproviders/status - relationaldatabaseproviders/status verbs: - get diff --git a/test-resources/test-suite.kind-config.yaml b/test-resources/test-suite.kind-config.yaml new file mode 100644 index 0000000..18eb9ae --- /dev/null +++ b/test-resources/test-suite.kind-config.yaml @@ -0,0 +1,2 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 62134c8..bed2ead 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -48,7 +48,7 @@ var _ = Describe("controller", Ordered, func() { Expect(utils.InstallMongoDB()).To(Succeed()) By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) + cmd := exec.Command(utils.Kubectl(), "create", "ns", namespace) _, _ = utils.Run(cmd) }) @@ -62,7 +62,7 @@ var _ = Describe("controller", Ordered, func() { By("removing the RelationalDatabaseProvider resource") for _, name := range []string{"mysql", "mysql-scope", "postgres", "mongodb"} { cmd := exec.Command( - "kubectl", + utils.Kubectl(), "patch", "relationaldatabaseprovider", fmt.Sprintf("relationaldatabaseprovider-%s-sample", name), @@ -72,14 +72,14 @@ var _ = Describe("controller", Ordered, func() { ) _, _ = utils.Run(cmd) cmd = exec.Command( - "kubectl", "delete", "--force", "relationaldatabaseprovider", fmt.Sprintf( + utils.Kubectl(), "delete", "--force", "relationaldatabaseprovider", fmt.Sprintf( "relationaldatabaseprovider-%s-sample", name)) _, _ = utils.Run(cmd) } By("removing the DatabaseRequest resource") for _, name := range []string{"mysql", "mysql-scope", "postgres", "seed"} { cmd := exec.Command( - "kubectl", + utils.Kubectl(), "patch", "databaserequest", fmt.Sprintf("databaserequest-%s-sample", name), @@ -89,12 +89,12 @@ var _ = Describe("controller", Ordered, func() { ) _, _ = utils.Run(cmd) cmd = exec.Command( - "kubectl", "delete", "--force", "databaserequest", fmt.Sprintf( + utils.Kubectl(), "delete", "--force", "databaserequest", fmt.Sprintf( "databaserequest-%s-sample", name)) _, _ = utils.Run(cmd) } By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) + cmd := exec.Command(utils.Kubectl(), "delete", "ns", namespace) _, _ = utils.Run(cmd) By("uninstalling relational databases pods") @@ -106,10 +106,10 @@ var _ = Describe("controller", Ordered, func() { By("removing service and secret") for _, name := range []string{"mysql", "mysql-scope", "postgres", "mongodb"} { cmd = exec.Command( - "kubectl", "delete", "service", "-n", "default", "-l", "app.kubernetes.io/instance=databaserequest-"+name+"-sample") + utils.Kubectl(), "delete", "service", "-n", "default", "-l", "app.kubernetes.io/instance=databaserequest-"+name+"-sample") _, _ = utils.Run(cmd) cmd = exec.Command( - "kubectl", "delete", "secret", "-n", "default", "-l", "app.kubernetes.io/instance=databaserequest-"+name+"-sample") + utils.Kubectl(), "delete", "secret", "-n", "default", "-l", "app.kubernetes.io/instance=databaserequest-"+name+"-sample") _, _ = utils.Run(cmd) } }) @@ -145,7 +145,7 @@ var _ = Describe("controller", Ordered, func() { verifyControllerUp := func() error { // Get pod name - cmd = exec.Command("kubectl", "get", + cmd = exec.Command(utils.Kubectl(), "get", "pods", "-l", "control-plane=controller-manager", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ @@ -164,7 +164,7 @@ var _ = Describe("controller", Ordered, func() { ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) // Validate pod status - cmd = exec.Command("kubectl", "get", + cmd = exec.Command(utils.Kubectl(), "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) @@ -182,7 +182,7 @@ var _ = Describe("controller", Ordered, func() { if name != "seed" { By("creating a RelationalDatabaseProvider resource") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "apply", "-f", fmt.Sprintf("config/samples/crd_v1alpha1_relationaldatabaseprovider_%s.yaml", name), @@ -192,7 +192,7 @@ var _ = Describe("controller", Ordered, func() { By("validating that the RelationalDatabaseProvider resource is created") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "wait", "--for=condition=Ready", "relationaldatabaseprovider", @@ -203,12 +203,12 @@ var _ = Describe("controller", Ordered, func() { ExpectWithOffset(1, err).NotTo(HaveOccurred()) } else { By("creating seed secret for the DatabaseRequest resource") - cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/testdata/seed-secret.yaml") + cmd = exec.Command(utils.Kubectl(), "apply", "-f", "test/e2e/testdata/seed-secret.yaml") _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("creating mysql pod to create seed credentials") - cmd = exec.Command("kubectl", "apply", "-f", "test/e2e/testdata/mysql-client-pod.yaml") + cmd = exec.Command(utils.Kubectl(), "apply", "-f", "test/e2e/testdata/mysql-client-pod.yaml") _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -220,7 +220,7 @@ var _ = Describe("controller", Ordered, func() { By("creating a DatabaseRequest resource") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "-n", "default", "apply", "-f", @@ -231,7 +231,7 @@ var _ = Describe("controller", Ordered, func() { By("validating that the DatabaseRequest resource is created") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "-n", "default", "wait", "--for=condition=Ready", @@ -245,7 +245,7 @@ var _ = Describe("controller", Ordered, func() { // verify that the service and secret got created By("validating that the service is created") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "get", "service", "-n", "default", @@ -259,7 +259,7 @@ var _ = Describe("controller", Ordered, func() { By("validating that the secret is created") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "get", "secret", "-n", "default", @@ -272,7 +272,7 @@ var _ = Describe("controller", Ordered, func() { if name == "seed" { By("checking that the seed secret is deleted") - cmd = exec.Command("kubectl", "get", "secret", "seed-mysql-secret") + cmd = exec.Command(utils.Kubectl(), "get", "secret", "seed-mysql-secret") _, err := utils.Run(cmd) // expect error to occurred ExpectWithOffset(1, err).To(HaveOccurred()) @@ -280,7 +280,7 @@ var _ = Describe("controller", Ordered, func() { By("deleting the DatabaseRequest resource the database is getting deprovisioned") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "-n", "default", "delete", "databaserequest", @@ -291,7 +291,7 @@ var _ = Describe("controller", Ordered, func() { By("validating that the service is deleted") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "get", "service", "-n", "default", @@ -304,7 +304,7 @@ var _ = Describe("controller", Ordered, func() { By("validating that the secret is deleted") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "get", "secret", "-n", "default", @@ -319,7 +319,7 @@ var _ = Describe("controller", Ordered, func() { By("validating that broken seed database request are failing in exptected way") for _, name := range []string{"credential-broken-seed", "non-existing-database-seed"} { By("creating seed secret for the DatabaseRequest resource") - cmd = exec.Command("kubectl", "apply", "-f", fmt.Sprintf("test/e2e/testdata/%s-secret.yaml", name)) + cmd = exec.Command(utils.Kubectl(), "apply", "-f", fmt.Sprintf("test/e2e/testdata/%s-secret.yaml", name)) _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -327,7 +327,7 @@ var _ = Describe("controller", Ordered, func() { // replace - with _ dbrName := strings.ReplaceAll(name, "-", "_") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "-n", "default", "apply", "-f", @@ -338,7 +338,7 @@ var _ = Describe("controller", Ordered, func() { By("validating that the DatabaseRequest resource is created but fails") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "-n", "default", "get", "databaserequest", @@ -362,7 +362,7 @@ var _ = Describe("controller", Ordered, func() { // verify that the service and secret got created By("validating that the service is not created") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "get", "service", "-n", "default", @@ -374,7 +374,7 @@ var _ = Describe("controller", Ordered, func() { By("validating that the secret is not created") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "get", "secret", "-n", "default", @@ -385,7 +385,7 @@ var _ = Describe("controller", Ordered, func() { Expect(strings.TrimSpace(string(serviceOutput))).To(Equal("No resources found in default namespace.")) By("validating that the seed secret is not deleted") - cmd = exec.Command("kubectl", "get", "secret", fmt.Sprintf("%s-secret", name)) + cmd = exec.Command(utils.Kubectl(), "get", "secret", fmt.Sprintf("%s-secret", name)) _, err = utils.Run(cmd) // expect no error to have occurred because the secret should not get deleted // if the database request is not successfully created @@ -393,7 +393,7 @@ var _ = Describe("controller", Ordered, func() { By("deleting the DatabaseRequest resource the database is getting deprovisioned") cmd = exec.Command( - "kubectl", + utils.Kubectl(), "-n", "default", "delete", "databaserequest", diff --git a/test/utils/utils.go b/test/utils/utils.go index c1dd070..5f44b85 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -37,6 +37,26 @@ const ( postgresYaml = "test/e2e/testdata/postgres.yaml" ) +var kubectlPath, kindPath string + +func init() { + if v, ok := os.LookupEnv("KIND_PATH"); ok { + kindPath = v + } else { + kindPath = "kind" + } + if v, ok := os.LookupEnv("KUBECTL_PATH"); ok { + kubectlPath = v + } else { + kubectlPath = "kubectl" + } + fmt.Println(kubectlPath, kindPath) +} + +func Kubectl() string { + return kubectlPath +} + func warnError(err error) { fmt.Fprintf(ginkgo.GinkgoWriter, "warning: %v\n", err) } @@ -49,7 +69,7 @@ func InstallRelationalDatabases() error { } errChan := make(chan error, 2) for _, yaml := range []string{mysqlYaml, postgresYaml} { - cmd := exec.Command("kubectl", "apply", "-f", yaml) + cmd := exec.Command(kubectlPath, "apply", "-f", yaml) cmd.Dir = dir fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s in directory: %s\n", strings.Join(cmd.Args, " "), dir) go func() { @@ -71,7 +91,7 @@ func InstallMongoDB() error { if err != nil { return err } - cmd := exec.Command("kubectl", "apply", "-f", "test/e2e/testdata/mongodb.yaml") + cmd := exec.Command(kubectlPath, "apply", "-f", "test/e2e/testdata/mongodb.yaml") cmd.Dir = dir fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s in directory: %s\n", strings.Join(cmd.Args, " "), dir) _, err = Run(cmd) @@ -86,7 +106,7 @@ func UninstallRelationalDatabases() { } errChan := make(chan error, 2) for _, yaml := range []string{mysqlYaml, postgresYaml} { - cmd := exec.Command("kubectl", "delete", "-f", yaml) + cmd := exec.Command(kubectlPath, "delete", "-f", yaml) cmd.Dir = dir fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s in directory: %s\n", strings.Join(cmd.Args, " "), dir) go func() { @@ -107,7 +127,7 @@ func UninstallMongoDB() { if err != nil { warnError(err) } - cmd := exec.Command("kubectl", "delete", "-f", "test/e2e/testdata/mongodb.yaml") + cmd := exec.Command(kubectlPath, "delete", "-f", "test/e2e/testdata/mongodb.yaml") cmd.Dir = dir if _, err := Run(cmd); err != nil { warnError(err) @@ -120,7 +140,7 @@ func InstallMySQL() error { if err != nil { return err } - cmd := exec.Command("kubectl", "apply", "-f", mysqlYaml) + cmd := exec.Command(kubectlPath, "apply", "-f", mysqlYaml) cmd.Dir = dir fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s in directory: %s\n", strings.Join(cmd.Args, " "), dir) _, err = Run(cmd) @@ -135,7 +155,7 @@ func UninstallMySQLPod() { if err != nil { warnError(err) } - cmd := exec.Command("kubectl", "delete", "-f", mysqlYaml) + cmd := exec.Command(kubectlPath, "delete", "-f", mysqlYaml) cmd.Dir = dir if _, err := Run(cmd); err != nil { warnError(err) @@ -145,7 +165,7 @@ func UninstallMySQLPod() { // InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. func InstallPrometheusOperator() error { url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "create", "-f", url) + cmd := exec.Command(kubectlPath, "create", "-f", url) _, err := Run(cmd) return err } @@ -173,7 +193,7 @@ func Run(cmd *exec.Cmd) ([]byte, error) { // UninstallPrometheusOperator uninstalls the prometheus func UninstallPrometheusOperator() { url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) + cmd := exec.Command(kubectlPath, "delete", "-f", url) if _, err := Run(cmd); err != nil { warnError(err) } @@ -182,7 +202,7 @@ func UninstallPrometheusOperator() { // UninstallCertManager uninstalls the cert manager func UninstallCertManager() { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) + cmd := exec.Command(kubectlPath, "delete", "-f", url) if _, err := Run(cmd); err != nil { warnError(err) } @@ -191,13 +211,13 @@ func UninstallCertManager() { // InstallCertManager installs the cert manager bundle. func InstallCertManager() error { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "apply", "-f", url) + cmd := exec.Command(kubectlPath, "apply", "-f", url) if _, err := Run(cmd); err != nil { return err } // Wait for cert-manager-webhook to be ready, which can take time if cert-manager // was re-installed after uninstalling on a cluster. - cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", + cmd = exec.Command(kubectlPath, "wait", "deployment.apps/cert-manager-webhook", "--for", "condition=Available", "--namespace", "cert-manager", "--timeout", "5m", @@ -209,12 +229,12 @@ func InstallCertManager() error { // LoadImageToKindCluster loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error { - cluster := "kind" + cluster := kindPath if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { cluster = v } kindOptions := []string{"load", "docker-image", name, "--name", cluster} - cmd := exec.Command("kind", kindOptions...) + cmd := exec.Command(kindPath, kindOptions...) _, err := Run(cmd) return err }