diff --git a/.github/workflows/e2e-linux.yaml b/.github/workflows/e2e-linux.yaml index 8b1f5e10e..a451c2356 100644 --- a/.github/workflows/e2e-linux.yaml +++ b/.github/workflows/e2e-linux.yaml @@ -120,6 +120,8 @@ jobs: sudo -E env "PATH=$PATH" INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make test-e2e-container sudo -E env "PATH=$PATH" INSTALLED=true REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make test-e2e-vm sudo -E env "PATH=/usr/libexec/finch:/usr/libexec/finch/cni/bin/:/usr/local/bin:$PATH" NERDCTL_TOML=/etc/finch/nerdctl/nerdctl.toml BUILDKIT_HOST=unix:///var/lib/finch/buildkit/buildkitd.sock make test-e2e-daemon-linux + - name: Show test coverage + run: make test-e2e-cov - name: Change ownership of reports if: always() run: | diff --git a/.github/workflows/e2e-macos.yaml b/.github/workflows/e2e-macos.yaml index d3ca99cc1..21611ecd5 100644 --- a/.github/workflows/e2e-macos.yaml +++ b/.github/workflows/e2e-macos.yaml @@ -93,6 +93,8 @@ jobs: git clean -f -d REGISTRY=${{ steps.vars.outputs.has_creds == true && env.REGISTRY || '' }} make ${{ inputs.test-command }} shell: zsh {0} + - name: Show test coverage + run: make test-e2e-cov - name: Set artifacts name outputs if: always() id: set-multiple-vars diff --git a/.github/workflows/e2e-ubuntu-finch.yaml b/.github/workflows/e2e-ubuntu-finch.yaml index 4f1475284..4b9d679f1 100644 --- a/.github/workflows/e2e-ubuntu-finch.yaml +++ b/.github/workflows/e2e-ubuntu-finch.yaml @@ -175,6 +175,8 @@ jobs: eval $(ssh-agent) sudo -E env "PATH=$PATH" INSTALLED=true make test-e2e-container sudo -E env "PATH=$PATH" INSTALLED=true make test-e2e-vm + - name: Show test coverage + run: make test-e2e-cov - name: Change ownership of reports if: always() run: | diff --git a/.github/workflows/e2e-windows.yaml b/.github/workflows/e2e-windows.yaml index 009ed92cd..3ef4d2048 100644 --- a/.github/workflows/e2e-windows.yaml +++ b/.github/workflows/e2e-windows.yaml @@ -98,6 +98,8 @@ jobs: git clean -f -d make ${{ inputs.test-command }} + - name: Show test coverage + run: make test-e2e-cov - name: Set artifacts name outputs if: always() id: set-multiple-vars diff --git a/.gitignore b/.gitignore index 709377370..9aabdbdc5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ _vde_output/ **/*.tar.gz **/*.lz4 **/*.img +cov/ tmp/ .vscode/ tools_bin/ diff --git a/Makefile b/Makefile index 75da162e2..744306f6a 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ DEST := $(shell echo "$(DESTDIR)/$(PREFIX)" | sed 's:///*:/:g; s://*$$::') BINDIR ?= /usr/local/bin OUTDIR ?= $(CURDIR)/_output OS_OUTDIR ?= $(OUTDIR)/os +COVERAGE_DIR ?= $(CURDIR)/cov REPORT_DIR ?= $(CURDIR)/reports RUN_ID ?= $(GITHUB_RUN_ID) RUN_ATTEMPT ?= $(GITHUB_RUN_ATTEMPT) @@ -177,7 +178,7 @@ $(OUTDIR)/finch-daemon/finch@.service: .PHONY: coverage coverage: - go test $(shell go list ./... | grep -v e2e | grep -v benchmark | grep -v mocks) -coverprofile=test-coverage.out + go test $(shell go list ./... | grep -v e2e | grep -v benchmark | grep -v mocks) go tool cover -html=test-coverage.out .PHONY: download-licenses @@ -291,6 +292,10 @@ test-unit: # Container tests and VM tests can be run in any order, but they must be run sequentially. # For more details, see the package-level comment of the e2e package. +.PHONY: create-coverage-dir +create-coverage-dir: + mkdir -p $(COVERAGE_DIR) + .PHONY: create-report-dir create-report-dir: mkdir -p $(REPORT_DIR) @@ -299,16 +304,20 @@ create-report-dir: test-e2e: test-e2e-vm-serial test-e2e-container .PHONY: test-e2e-vm-serial -test-e2e-vm-serial: create-report-dir - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-serial-report.json --installed="$(INSTALLED)" +test-e2e-vm-serial: create-report-dir create-coverage-dir + go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-serial-report.json --installed="$(INSTALLED)" .PHONY: test-e2e-container -test-e2e-container: create-report-dir - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/container -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-container-report.json --installed="$(INSTALLED)" +test-e2e-container: create-report-dir create-coverage-dir + go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/container -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-container-report.json --installed="$(INSTALLED)" .PHONY: test-e2e-vm -test-e2e-vm: create-report-dir - go test -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -ginkgo.v -ginkgo.timeout=2h -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-report.json --installed="$(INSTALLED)" --registry="$(REGISTRY)" +test-e2e-vm: create-report-dir create-coverage-dir + go test -coverpkg=./... -ldflags $(LDFLAGS) -timeout 2h ./e2e/vm -test.v -test.gocoverdir=$(COVERAGE_DIR) -ginkgo.v -ginkgo.timeout=2h -ginkgo.focus "updates init-only config values when values are changed after init" -ginkgo.flake-attempts=3 -ginkgo.json-report=$(REPORT_DIR)/$(RUN_ID)-$(RUN_ATTEMPT)-e2e-vm-report.json --installed="$(INSTALLED)" --registry="$(REGISTRY)" + +.PHONY: test-e2e-cov +test-e2e-cov: create-coverage-dir + @go tool covdata percent -i=$(COVERAGE_DIR) -pkg=$(shell go list ./... | grep -v e2e | grep -v benchmark | grep -v version | tr '\n' ',') GINKGO = go run github.com/onsi/ginkgo/v2/ginkgo # Common ginkgo options: -v for verbose mode, --focus="test name" for running single tests