From 837b3f295c317261584681fda29f88c718103953 Mon Sep 17 00:00:00 2001 From: gounux Date: Sun, 22 Dec 2024 11:38:03 +0100 Subject: [PATCH 01/17] Use docker's build-push-action to speedup tests CI --- .github/workflows/test.yml | 56 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc66dc6a8..c7704b47a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,12 +61,64 @@ jobs: sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml/g' .env eval $(egrep "^[^#;]" .env | xargs -d'\n' -n1 | sed -E 's/(\w+)=(.*)/export \1='"'"'\2'"'"'/g') + - name: Login to GitHub Container Repository (ghcr.io) + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Pull docker containers run: docker compose pull + - name: Docker Build app + uses: docker/build-push-action@v6 + with: + file: docker-app/Dockerfile + context: docker-app + push: false + load: true + target: webserver_test + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache,mode=max + + - name: Docker Build worker-wrapper + uses: docker/build-push-action@v6 + with: + file: docker-app/Dockerfile + context: docker-app + push: false + load: true + target: worker_wrapper_runtime + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache,mode=max + + - name: Docker Build qgis + uses: docker/build-push-action@v6 + with: + file: docker-qgis/Dockerfile + context: docker-qgis + push: false + load: true + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache,mode=max + + - name: Docker Build nginx + uses: docker/build-push-action@v6 + with: + file: docker-nginx/Dockerfile + context: docker-nginx + push: false + load: true + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache,mode=max + - name: Build and run docker containers - run: | - docker compose up -d --build + run: docker compose up -d - name: Initial manage.py commands run: | From 893cbb04a24e8328e60eec5cedb37b782a687967 Mon Sep 17 00:00:00 2001 From: gounux Date: Sun, 22 Dec 2024 12:20:14 +0100 Subject: [PATCH 02/17] Add images local tags --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c7704b47a..0d216dfca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,6 +60,7 @@ jobs: cp .env.example .env sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml/g' .env eval $(egrep "^[^#;]" .env | xargs -d'\n' -n1 | sed -E 's/(\w+)=(.*)/export \1='"'"'\2'"'"'/g') + cat .env - name: Login to GitHub Container Repository (ghcr.io) uses: docker/login-action@v3 @@ -83,6 +84,7 @@ jobs: push: false load: true target: webserver_test + tags: qfieldcloud-app:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache,mode=max @@ -94,6 +96,7 @@ jobs: push: false load: true target: worker_wrapper_runtime + tags: qfieldcloud-worker_wrapper:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache,mode=max @@ -104,6 +107,7 @@ jobs: context: docker-qgis push: false load: true + tags: qfieldcloud-qgis:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache,mode=max @@ -114,10 +118,11 @@ jobs: context: docker-nginx push: false load: true + tags: qfieldcloud-nginx:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache,mode=max - - name: Build and run docker containers + - name: Run docker containers run: docker compose up -d - name: Initial manage.py commands From f29ef27d9e383f3e76627838aee2f2403cfc4c32 Mon Sep 17 00:00:00 2001 From: gounux Date: Thu, 26 Dec 2024 15:28:56 +0100 Subject: [PATCH 03/17] Use app-test buildcache for app remote caching --- .github/workflows/test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d216dfca..5936649fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,6 @@ jobs: cp .env.example .env sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml/g' .env eval $(egrep "^[^#;]" .env | xargs -d'\n' -n1 | sed -E 's/(\w+)=(.*)/export \1='"'"'\2'"'"'/g') - cat .env - name: Login to GitHub Container Repository (ghcr.io) uses: docker/login-action@v3 @@ -85,8 +84,8 @@ jobs: load: true target: webserver_test tags: qfieldcloud-app:latest - cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache - # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache,mode=max + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache,mode=max - name: Docker Build worker-wrapper uses: docker/build-push-action@v6 From 27e84725eafa07f6634986d1c548572d913e5d59 Mon Sep 17 00:00:00 2001 From: gounux Date: Thu, 26 Dec 2024 15:36:59 +0100 Subject: [PATCH 04/17] Try rebuilding images in compose --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5936649fe..a404aefe7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -122,7 +122,7 @@ jobs: # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache,mode=max - name: Run docker containers - run: docker compose up -d + run: docker compose up -d --build - name: Initial manage.py commands run: | From 31bcfc96ae28d55aa451f89556dc3e49b8e5b552 Mon Sep 17 00:00:00 2001 From: gounux Date: Thu, 26 Dec 2024 15:49:53 +0100 Subject: [PATCH 05/17] Allow network.host on required images --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a404aefe7..9601357ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,6 +71,8 @@ jobs: - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host - name: Pull docker containers run: docker compose pull @@ -82,6 +84,7 @@ jobs: context: docker-app push: false load: true + allow: network.host target: webserver_test tags: qfieldcloud-app:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache @@ -94,6 +97,7 @@ jobs: context: docker-app push: false load: true + allow: network.host target: worker_wrapper_runtime tags: qfieldcloud-worker_wrapper:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache @@ -106,6 +110,7 @@ jobs: context: docker-qgis push: false load: true + allow: network.host tags: qfieldcloud-qgis:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache,mode=max @@ -122,7 +127,7 @@ jobs: # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache,mode=max - name: Run docker containers - run: docker compose up -d --build + run: docker compose up -d - name: Initial manage.py commands run: | From 16f8d68b33e301d3cc94bb3db7be00748ed524c3 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 17:28:56 +0100 Subject: [PATCH 06/17] Build docker images with bake --- .github/workflows/test.yml | 52 ++++++-------------------------------- 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9601357ea..2fa3cc6d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,59 +72,23 @@ jobs: id: buildx uses: docker/setup-buildx-action@v3 with: + buildkitd-flags: '--allow-insecure-entitlement network.host' driver-opts: network=host - name: Pull docker containers run: docker compose pull - - name: Docker Build app - uses: docker/build-push-action@v6 + - name: Build docker images locally + uses: docker/bake-action@v5 with: - file: docker-app/Dockerfile - context: docker-app push: false load: true allow: network.host - target: webserver_test - tags: qfieldcloud-app:latest - cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache - # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache,mode=max - - - name: Docker Build worker-wrapper - uses: docker/build-push-action@v6 - with: - file: docker-app/Dockerfile - context: docker-app - push: false - load: true - allow: network.host - target: worker_wrapper_runtime - tags: qfieldcloud-worker_wrapper:latest - cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache - # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache,mode=max - - - name: Docker Build qgis - uses: docker/build-push-action@v6 - with: - file: docker-qgis/Dockerfile - context: docker-qgis - push: false - load: true - allow: network.host - tags: qfieldcloud-qgis:latest - cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache - # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache,mode=max - - - name: Docker Build nginx - uses: docker/build-push-action@v6 - with: - file: docker-nginx/Dockerfile - context: docker-nginx - push: false - load: true - tags: qfieldcloud-nginx:latest - cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache - # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache,mode=max + set: | + app.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache + worker_wrapper.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache + qgis.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache + nginx.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache - name: Run docker containers run: docker compose up -d From 01762bb48a60b3daa4f878a66867d4bdbae1c5cf Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 18:06:58 +0100 Subject: [PATCH 07/17] Expose gh runtime --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2fa3cc6d6..21d29b6ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,6 +78,12 @@ jobs: - name: Pull docker containers run: docker compose pull + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v3 + + - name: Print bake config + run: docker buildx bake --print + - name: Build docker images locally uses: docker/bake-action@v5 with: From 5f2a9be3a1354ddf8c87e1aa1cf46b9313828174 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 18:18:02 +0100 Subject: [PATCH 08/17] Set docker image tags --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 21d29b6ae..7788a1cad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -92,9 +92,16 @@ jobs: allow: network.host set: | app.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache + app.tags=qfielcloud-app:latest worker_wrapper.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache + worker_wrapper.tags=qfielcloud-worker_wrapper:latest qgis.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache + qgis.tags=qfielcloud-qgis:latest nginx.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache + nginx.tags=qfielcloud-nginx:latest + env: + DOCKER_BUILD_RECORD_UPLOAD: false + - name: Run docker containers run: docker compose up -d From 3416eede4cf59e177a9ded7b78e71340397d1bde Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 18:54:32 +0100 Subject: [PATCH 09/17] Use already built image for tests --- .github/workflows/test.yml | 36 ++++++++++++++-------------------- docker-compose.override.ci.yml | 10 ++++++++++ 2 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 docker-compose.override.ci.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7788a1cad..a6569c011 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,11 @@ on: - master - release pull_request: + workflow_run: + workflows: + - Build and Push + types: + - completed jobs: lint: @@ -35,6 +40,7 @@ jobs: test: name: Run tests runs-on: ubuntu-24.04 + if: ${{ github.event.workflow_run.conclusion == 'success' }} strategy: fail-fast: false matrix: @@ -58,8 +64,9 @@ jobs: - name: Export the env variables file run: | cp .env.example .env - sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml/g' .env + sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml:docker-compose.override.ci.yml/g' .env eval $(egrep "^[^#;]" .env | xargs -d'\n' -n1 | sed -E 's/(\w+)=(.*)/export \1='"'"'\2'"'"'/g') + cat .env - name: Login to GitHub Container Repository (ghcr.io) uses: docker/login-action@v3 @@ -78,29 +85,16 @@ jobs: - name: Pull docker containers run: docker compose pull - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v3 - - - name: Print bake config - run: docker buildx bake --print - - - name: Build docker images locally - uses: docker/bake-action@v5 + - name: Build docker app-test image + uses: docker/build-push-action@v6 with: + file: docker-app/Dockerfile + context: docker-app push: false load: true - allow: network.host - set: | - app.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache - app.tags=qfielcloud-app:latest - worker_wrapper.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache - worker_wrapper.tags=qfielcloud-worker_wrapper:latest - qgis.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache - qgis.tags=qfielcloud-qgis:latest - nginx.cache-from=type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache - nginx.tags=qfielcloud-nginx:latest - env: - DOCKER_BUILD_RECORD_UPLOAD: false + target: webserver_test + tags: qfieldcloud-app:latest + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache - name: Run docker containers diff --git a/docker-compose.override.ci.yml b/docker-compose.override.ci.yml new file mode 100644 index 000000000..d63b1cce8 --- /dev/null +++ b/docker-compose.override.ci.yml @@ -0,0 +1,10 @@ +services: + + worker_wrapper: + image: ghcr.io/opengisch/qfieldcloud-worker-wrapper:commit-${GITHUB_SHA} + + qgis: + image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA} + + nginx: + image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA} From f330b5dffd14814616f78b98b6e08769bffd2c6f Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 18:57:47 +0100 Subject: [PATCH 10/17] Edit test workflow condition --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a6569c011..4d5dbbfb5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,6 @@ jobs: test: name: Run tests runs-on: ubuntu-24.04 - if: ${{ github.event.workflow_run.conclusion == 'success' }} strategy: fail-fast: false matrix: From e20e1d60ca4703836680380dcfab1f827c665ac3 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 19:05:57 +0100 Subject: [PATCH 11/17] Wait for build to finish before starting tests --- .github/workflows/test.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d5dbbfb5..d231f2f0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -81,9 +81,6 @@ jobs: buildkitd-flags: '--allow-insecure-entitlement network.host' driver-opts: network=host - - name: Pull docker containers - run: docker compose pull - - name: Build docker app-test image uses: docker/build-push-action@v6 with: @@ -95,6 +92,16 @@ jobs: tags: qfieldcloud-app:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache + - name: Wait for parallel build to finish + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ github.ref }} + check-name: 'Build and Push' + repo-token: ${{ secrets.GITHUB_TOKEN }} + wait-interval: 5 + + - name: Pull docker containers + run: docker compose pull - name: Run docker containers run: docker compose up -d From 1f442db4d5cb9010d71dba2a35ca9a8f05bc5c52 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 19:09:39 +0100 Subject: [PATCH 12/17] Do not wait on build to finish before starting tests --- .github/workflows/test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d231f2f0f..387ad8a38 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -92,13 +92,13 @@ jobs: tags: qfieldcloud-app:latest cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache - - name: Wait for parallel build to finish - uses: lewagon/wait-on-check-action@v1.3.4 - with: - ref: ${{ github.ref }} - check-name: 'Build and Push' - repo-token: ${{ secrets.GITHUB_TOKEN }} - wait-interval: 5 + # - name: Wait for parallel build to finish + # uses: lewagon/wait-on-check-action@v1.3.4 + # with: + # ref: ${{ github.ref }} + # check-name: 'Build and Push' + # repo-token: ${{ secrets.GITHUB_TOKEN }} + # wait-interval: 5 - name: Pull docker containers run: docker compose pull From dd013586052d85539b11aeabf14a0c9023a132ed Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 19:16:43 +0100 Subject: [PATCH 13/17] Reset build in docker compose ci override --- docker-compose.override.ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.override.ci.yml b/docker-compose.override.ci.yml index d63b1cce8..dce79311f 100644 --- a/docker-compose.override.ci.yml +++ b/docker-compose.override.ci.yml @@ -2,9 +2,12 @@ services: worker_wrapper: image: ghcr.io/opengisch/qfieldcloud-worker-wrapper:commit-${GITHUB_SHA} + build: "!reset null" qgis: image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA} + build: "!reset null" nginx: image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA} + build: "!reset null" From 951154075ac52c1959b936192d8f054eb28d6543 Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 19:24:51 +0100 Subject: [PATCH 14/17] Reset compose build commands in ci override --- .pre-commit-config.yaml | 1 + docker-compose.override.ci.yml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f3636268..f98a47313 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,7 @@ repos: - id: check-symlinks - id: check-toml - id: check-yaml + args: [--unsafe] - id: debug-statements - id: destroyed-symlinks - id: end-of-file-fixer diff --git a/docker-compose.override.ci.yml b/docker-compose.override.ci.yml index dce79311f..cbe91283c 100644 --- a/docker-compose.override.ci.yml +++ b/docker-compose.override.ci.yml @@ -2,12 +2,12 @@ services: worker_wrapper: image: ghcr.io/opengisch/qfieldcloud-worker-wrapper:commit-${GITHUB_SHA} - build: "!reset null" + build: !reset null qgis: image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA} - build: "!reset null" + build: !reset null nginx: image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA} - build: "!reset null" + build: !reset null From c26f0cff420162adf6b1ef1312488faf2e14a92f Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 19:43:08 +0100 Subject: [PATCH 15/17] Use short GITHUB_SHA --- .github/workflows/test.yml | 5 +++++ docker-compose.override.ci.yml | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 387ad8a38..e36b35ff2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,6 +100,11 @@ jobs: # repo-token: ${{ secrets.GITHUB_TOKEN }} # wait-interval: 5 + - name: Debug + run: | + echo ${GITHUB_SHA} + echo ${GITHUB_SHA:0:7} + - name: Pull docker containers run: docker compose pull diff --git a/docker-compose.override.ci.yml b/docker-compose.override.ci.yml index cbe91283c..de0740769 100644 --- a/docker-compose.override.ci.yml +++ b/docker-compose.override.ci.yml @@ -1,13 +1,13 @@ services: worker_wrapper: - image: ghcr.io/opengisch/qfieldcloud-worker-wrapper:commit-${GITHUB_SHA} + image: ghcr.io/opengisch/qfieldcloud-worker-wrapper:commit-${GITHUB_SHA:0:7} build: !reset null qgis: - image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA} + image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA:0:7} build: !reset null nginx: - image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA} + image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA:0:7} build: !reset null From 3e5777bb415245adbc511af09931e5f9ca2f9a3c Mon Sep 17 00:00:00 2001 From: gounux Date: Fri, 27 Dec 2024 19:48:50 +0100 Subject: [PATCH 16/17] Use short sha for ci compose images override --- .github/workflows/test.yml | 8 +++----- docker-compose.override.ci.yml | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e36b35ff2..28091a37a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,9 @@ on: types: - completed +env: + SHORT_SHA: ${GITHUB_SHA:0:7} + jobs: lint: if: '! github.event.pull_request.draft' @@ -100,11 +103,6 @@ jobs: # repo-token: ${{ secrets.GITHUB_TOKEN }} # wait-interval: 5 - - name: Debug - run: | - echo ${GITHUB_SHA} - echo ${GITHUB_SHA:0:7} - - name: Pull docker containers run: docker compose pull diff --git a/docker-compose.override.ci.yml b/docker-compose.override.ci.yml index de0740769..b6cc475e6 100644 --- a/docker-compose.override.ci.yml +++ b/docker-compose.override.ci.yml @@ -1,13 +1,13 @@ services: worker_wrapper: - image: ghcr.io/opengisch/qfieldcloud-worker-wrapper:commit-${GITHUB_SHA:0:7} + image: ghcr.io/opengisch/qfieldcloud-worker-wrapper:commit-${SHORT_SHA} build: !reset null qgis: - image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA:0:7} + image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${SHORT_SHA} build: !reset null nginx: - image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${GITHUB_SHA:0:7} + image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${SHORT_SHA} build: !reset null From c680894371ff953f2215b1244153f427acd428a0 Mon Sep 17 00:00:00 2001 From: gounux Date: Sun, 29 Dec 2024 09:18:36 +0100 Subject: [PATCH 17/17] Use docker-build-push action to build images --- .github/workflows/test.yml | 60 +++++++++++++++++++++++++--------- .pre-commit-config.yaml | 1 - docker-compose.override.ci.yml | 13 -------- 3 files changed, 44 insertions(+), 30 deletions(-) delete mode 100644 docker-compose.override.ci.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28091a37a..1e7ce8801 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,11 +10,6 @@ on: - master - release pull_request: - workflow_run: - workflows: - - Build and Push - types: - - completed env: SHORT_SHA: ${GITHUB_SHA:0:7} @@ -66,9 +61,8 @@ jobs: - name: Export the env variables file run: | cp .env.example .env - sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml:docker-compose.override.ci.yml/g' .env + sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml/g' .env eval $(egrep "^[^#;]" .env | xargs -d'\n' -n1 | sed -E 's/(\w+)=(.*)/export \1='"'"'\2'"'"'/g') - cat .env - name: Login to GitHub Container Repository (ghcr.io) uses: docker/login-action@v3 @@ -84,24 +78,58 @@ jobs: buildkitd-flags: '--allow-insecure-entitlement network.host' driver-opts: network=host - - name: Build docker app-test image + - name: Docker Build app uses: docker/build-push-action@v6 with: file: docker-app/Dockerfile context: docker-app push: false load: true + allow: network.host + network: host target: webserver_test tags: qfieldcloud-app:latest - cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app:buildcache + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-app-test:buildcache,mode=max - # - name: Wait for parallel build to finish - # uses: lewagon/wait-on-check-action@v1.3.4 - # with: - # ref: ${{ github.ref }} - # check-name: 'Build and Push' - # repo-token: ${{ secrets.GITHUB_TOKEN }} - # wait-interval: 5 + - name: Docker Build worker-wrapper + uses: docker/build-push-action@v6 + with: + file: docker-app/Dockerfile + context: docker-app + push: false + load: true + allow: network.host + network: host + target: worker_wrapper_runtime + tags: qfieldcloud-worker_wrapper:latest + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-worker-wrapper:buildcache,mode=max + + - name: Docker Build qgis + uses: docker/build-push-action@v6 + with: + file: docker-qgis/Dockerfile + context: docker-qgis + push: false + load: true + allow: network.host + network: host + tags: qfieldcloud-qgis:latest + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-qgis:buildcache,mode=max + + - name: Docker Build nginx + uses: docker/build-push-action@v6 + with: + file: docker-nginx/Dockerfile + context: docker-nginx + push: false + load: true + allow: network.host + tags: qfieldcloud-nginx:latest + cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache + # cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-nginx:buildcache,mode=max - name: Pull docker containers run: docker compose pull diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f98a47313..8f3636268 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,6 @@ repos: - id: check-symlinks - id: check-toml - id: check-yaml - args: [--unsafe] - id: debug-statements - id: destroyed-symlinks - id: end-of-file-fixer diff --git a/docker-compose.override.ci.yml b/docker-compose.override.ci.yml deleted file mode 100644 index b6cc475e6..000000000 --- a/docker-compose.override.ci.yml +++ /dev/null @@ -1,13 +0,0 @@ -services: - - worker_wrapper: - image: ghcr.io/opengisch/qfieldcloud-worker-wrapper:commit-${SHORT_SHA} - build: !reset null - - qgis: - image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${SHORT_SHA} - build: !reset null - - nginx: - image: ghcr.io/opengisch/qfieldcloud-qgis:commit-${SHORT_SHA} - build: !reset null