Merge pull request #47 from uselagoon/database-test-e2e

test: add database tests with metrics checks

Author: shreddedbacon

.github/workflows/storage-calculator.yaml

@@ -42,6 +42,10 @@ jobs:
       with:
         go-version: '1.23'
 
+    - name: Add dependency chart repos
+      run: |
+        helm repo add amazeeio https://amazeeio.github.io/charts/
+
     - name: Create kind cluster
       uses: helm/kind-action@v1.12.0
       with:
@@ -54,6 +58,6 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
-    - name: Run test-e2e
+    - name: Run github/test-e2e
       run: |
-        make test-e2e KIND_NETWORK=kind
+        make github/test-e2e KIND_NETWORK=kind

Makefile

@@ -20,6 +20,8 @@ endif
 KIND_CLUSTER ?= storage-calculator
 KIND_NETWORK ?= storage-controller
 
+TIMEOUT = 30m
+
 KIND_VERSION = v0.25.0
 KUBECTL_VERSION := v1.31.0
 HELM_VERSION := v3.16.1
@@ -244,7 +246,7 @@ regenerate-broker-certs:
 
 # Create a kind cluster locally and run the test e2e test suite against it
 .PHONY: kind/test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up locally
-kind/test-e2e: create-kind-cluster kind/re-test-e2e
+kind/test-e2e: create-kind-cluster install-dbaas-operator kind/re-test-e2e
 
 .PHONY: local-kind/test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up locally
 kind/re-test-e2e:
@@ -260,12 +262,15 @@ kind/clean:
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up inside github action.
-test-e2e: local-dev/tools generate-broker-certs
+test-e2e: generate-broker-certs
 	($(KUBECTL) create namespace storage-calculator-system || echo "namespace exists") && \
 	($(KUBECTL) -n storage-calculator-system delete secret lagoon-broker-tls || echo "lagoon-broker-tls doesn't exist, ignoring") && \
 	$(KUBECTL) -n storage-calculator-system create secret generic lagoon-broker-tls --from-file=tls.crt=local-dev/certificates/clienttls.crt --from-file=tls.key=local-dev/certificates/clienttls.key --from-file=ca.crt=local-dev/certificates/ca.crt && \
 	go test ./test/e2e/ -v -ginkgo.v
 
+.PHONY: github/test-e2e
+github/test-e2e: local-dev/tools install-dbaas-operator test-e2e
+
 .PHONY: kind/set-kubeconfig
 kind/set-kubeconfig:
 	export KIND_CLUSTER=$(KIND_CLUSTER) && \
@@ -301,3 +306,56 @@ CONTROLLER_GEN=$(GOBIN)/controller-gen
 else
 CONTROLLER_GEN=$(shell which controller-gen)
 endif
+
+.PHONY: install-mariadb
+install-mariadb:
+	$(HELM) upgrade \
+		--install \
+		--create-namespace \
+		--namespace mariadb \
+		--wait \
+		--timeout $(TIMEOUT) \
+		$$($(KUBECTL) get ns mariadb > /dev/null 2>&1 && echo --set auth.rootPassword=$$($(KUBECTL) get secret --namespace mariadb mariadb -o json | $(JQ) -r '.data."mariadb-root-password" | @base64d')) \
+		--version=0.2.1 \
+		mariadb \
+		oci://registry-1.docker.io/cloudpirates/mariadb
+
+.PHONY: install-postgresql
+install-postgresql:
+	# root password is required on upgrade if the chart is already installed
+	$(HELM) upgrade \
+		--install \
+		--create-namespace \
+		--namespace postgresql \
+		--wait \
+		--timeout $(TIMEOUT) \
+		$$($(KUBECTL) get ns postgresql > /dev/null 2>&1 && echo --set auth.postgresPassword=$$($(KUBECTL) get secret --namespace postgresql postgresql -o json | $(JQ) -r '.data."postgres-password" | @base64d')) \
+		--version=0.2.2 \
+		postgresql \
+		oci://registry-1.docker.io/cloudpirates/postgres
+
+.PHONY: install-dbaas-operator
+install-dbaas-operator: install-mariadb install-postgresql
+	$(HELM) upgrade \
+		--install \
+		--create-namespace \
+		--namespace dbaas-operator \
+		--wait \
+		--timeout $(TIMEOUT) \
+		--set enableMariaDBProviders=true \
+		--set enableMongoDBProviders=true \
+		--set enablePostreSQLProviders=true \
+		--set mariadbProviders.development.environment=development \
+		--set mariadbProviders.development.hostname=mariadb.mariadb.svc.cluster.local \
+		--set mariadbProviders.development.readReplicaHostnames[0]=mariadb.mariadb.svc.cluster.local \
+		--set mariadbProviders.development.password=$$($(KUBECTL) get secret --namespace mariadb mariadb -o json | $(JQ) -r '.data."mariadb-root-password" | @base64d') \
+		--set mariadbProviders.development.port=3306 \
+		--set mariadbProviders.development.user=root \
+		--set postgresqlProviders.development.environment=development \
+		--set postgresqlProviders.development.hostname=postgresql.postgresql.svc.cluster.local \
+		--set postgresqlProviders.development.password=$$($(KUBECTL) get secret --namespace postgresql postgresql -o json | $(JQ) -r '.data."postgres-password" | @base64d') \
+		--set postgresqlProviders.development.port=5432 \
+		--set postgresqlProviders.development.user=postgres \
+		--version=0.4.0 \
+		dbaas-operator \
+		amazeeio/dbaas-operator

test/e2e/e2e_suite.go

@@ -37,8 +37,11 @@ var (
 	duration = 600 * time.Second
 	interval = 1 * time.Second
 
-	metricLabels = []string{
-		"lagoon_storage_calculator_kilobytes",
+	metricClaims = []string{
+		`lagoon_storage_calculator_kilobytes{claimenv="123",claimpvc="basic-data",environment="develop",lagoon_namespace="example-project-develop",project="example-project"}`,
+		`lagoon_storage_calculator_kilobytes{claimenv="123",claimpvc="basic-data",environment="main",lagoon_namespace="example-project-main",project="example-project"}`,
+		`lagoon_storage_calculator_kilobytes{claimenv="123",claimpvc="mariadb",environment="develop",lagoon_namespace="example-project-develop",project="example-project"}`,
+		`lagoon_storage_calculator_kilobytes{claimenv="123",claimpvc="postgres",environment="develop",lagoon_namespace="example-project-develop",project="example-project"}`,
 	}
 )
 
@@ -212,6 +215,21 @@ var _ = Describe("controller", Ordered, func() {
 				return nil
 			}
 			EventuallyWithOffset(1, verifyStorageCalculatorRuns, duration, interval).Should(Succeed())
+			verifyStorageCalculatorRuns2 := func() error {
+				cmd = exec.Command(utils.Kubectl(), "logs",
+					controllerPodName, "-c", "manager",
+					"-n", namespace,
+				)
+				podlogs, err := utils.Run(cmd)
+				ExpectWithOffset(2, err).NotTo(HaveOccurred())
+				if !strings.Contains(string(podlogs), "storage in example-project-develop") {
+					return fmt.Errorf("storage-calculator-pod not created")
+				}
+				return nil
+			}
+			EventuallyWithOffset(1, verifyStorageCalculatorRuns2, duration, interval).Should(Succeed())
+
+			time.Sleep(10 * time.Second)
 
 			By("validating that unauthenticated metrics requests fail")
 			runCmd := `curl -s -k https://storage-calculator-controller-manager-metrics-service.storage-calculator-system.svc.cluster.local:8443/metrics | grep -v "#" | grep "lagoon_"`
@@ -223,7 +241,7 @@ var _ = Describe("controller", Ordered, func() {
 			output, err := utils.RunCommonsCommand(namespace, runCmd)
 			ExpectWithOffset(2, err).NotTo(HaveOccurred())
 			fmt.Printf("metrics: %s", string(output))
-			err = utils.CheckStringContainsStrings(string(output), metricLabels)
+			err = utils.CheckStringContainsStrings(string(output), metricClaims)
 			ExpectWithOffset(2, err).NotTo(HaveOccurred())
 			// End tests
 		})

test/e2e/testdata/example-env-2.yaml

@@ -3,7 +3,7 @@ kind: Namespace
 metadata:
   name: example-project-develop
   labels:
-    lagoon.sh/environmentType: production
+    lagoon.sh/environmentType: development
     lagoon.sh/storageCalculatorEnabled: 'true'
     lagoon.sh/environmentId: '123'
     lagoon.sh/environment: 'develop'
@@ -22,7 +22,7 @@ metadata:
     app.kubernetes.io/name: basic-persistent
     lagoon.sh/buildType: branch
     lagoon.sh/environment: develop
-    lagoon.sh/environmentType: production
+    lagoon.sh/environmentType: development
     lagoon.sh/project: example-project
     lagoon.sh/service: basic
     lagoon.sh/service-type: basic-persistent
@@ -50,7 +50,7 @@ metadata:
     app.kubernetes.io/name: basic-persistent
     lagoon.sh/buildType: branch
     lagoon.sh/environment: develop
-    lagoon.sh/environmentType: production
+    lagoon.sh/environmentType: development
     lagoon.sh/project: example-project
     lagoon.sh/service: basic
     lagoon.sh/service-type: basic-persistent
@@ -77,7 +77,7 @@ spec:
         app.kubernetes.io/name: basic-persistent
         lagoon.sh/buildType: branch
         lagoon.sh/environment: develop
-        lagoon.sh/environmentType: production
+        lagoon.sh/environmentType: development
         lagoon.sh/project: example-project
         lagoon.sh/service: basic
         lagoon.sh/service-type: basic-persistent
@@ -135,7 +135,7 @@ metadata:
     app.kubernetes.io/name: basic-persistent
     lagoon.sh/buildType: branch
     lagoon.sh/environment: develop
-    lagoon.sh/environmentType: production
+    lagoon.sh/environmentType: development
     lagoon.sh/project: example-project
     lagoon.sh/service: basic
     lagoon.sh/service-type: basic-persistent
@@ -166,7 +166,7 @@ metadata:
     app.kubernetes.io/name: lagoon-env
     lagoon.sh/buildType: branch
     lagoon.sh/environment: name
-    lagoon.sh/environmentType: production
+    lagoon.sh/environmentType: development
     lagoon.sh/project: example-project
     lagoon.sh/template: lagoon-env-0.1.0
   name: lagoon-platform-env
@@ -186,12 +186,54 @@ metadata:
     app.kubernetes.io/name: lagoon-env
     lagoon.sh/buildType: branch
     lagoon.sh/environment: name
-    lagoon.sh/environmentType: production
+    lagoon.sh/environmentType: development
     lagoon.sh/project: example-project
     lagoon.sh/template: lagoon-env-0.1.0
   name: lagoon-env
   namespace: example-project-develop
 stringData:
   MY_SPECIAL_VARIABLE: myspecialvariable
   MY_SPECIAL_VARIABLE1: myspecialvariable1
-  MY_SPECIAL_VARIABLE2: myspecialvariable2
+  MY_SPECIAL_VARIABLE2: myspecialvariable2
+---
+apiVersion: mariadb.amazee.io/v1
+kind: MariaDBConsumer
+metadata:
+  annotations:
+    lagoon.sh/branch: environment-name
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: mariadb
+    app.kubernetes.io/managed-by: build-deploy-tool
+    app.kubernetes.io/name: mariadb
+    lagoon.sh/buildType: branch
+    lagoon.sh/environment: name
+    lagoon.sh/environmentType: development
+    lagoon.sh/project: example-project
+    lagoon.sh/service-type: mariadb-dbaas
+    lagoon.sh/template: mariadb-dbaas-0.1.0
+  name: mariadb
+  namespace: example-project-develop
+spec:
+  environment: development
+---
+apiVersion: postgres.amazee.io/v1
+kind: PostgreSQLConsumer
+metadata:
+  annotations:
+    lagoon.sh/branch: environment-name
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/instance: postgres
+    app.kubernetes.io/managed-by: build-deploy-tool
+    app.kubernetes.io/name: postgres
+    lagoon.sh/buildType: branch
+    lagoon.sh/environment: name
+    lagoon.sh/environmentType: development
+    lagoon.sh/project: example-project
+    lagoon.sh/service-type: postgres-dbaas
+    lagoon.sh/template: postgres-dbaas-0.1.0
+  name: postgres
+  namespace: example-project-develop
+spec:
+  environment: development