arm64 and kubernetes compatibility

.env.example

@@ -163,6 +163,21 @@ WEB_HTTPS_PORT=443
 # DEFAULT: error
 NGINX_ERROR_LOG_LEVEL=error
 
+# Sets the maximum allowed size of the client request body.
+# DEFAULT: 10g
+NGINX_CLIENT_MAX_BODY_SIZE=10g
+
+# Timeout for establishing a connection with the proxied server
+# DEFAULT: 5s
+NGINX_PROXY_CONNECT_TIMEOUT=5s
+
+# Timeout for reading a response from the proxied server.
+# DEFAULT: 300s
+NGINX_PROXY_READ_TIMEOUT=300s
+
+# Timeout for transmitting a request to the proxied server.
+# DEFAULT: 300s
+NGINX_PROXY_SEND_TIMEOUT=300s
 
 ##################
 # Gunicorn settings
@@ -449,8 +464,8 @@ SMTP4DEV_IMAP_PORT=143
 # see this gist: https://gist.github.com/gounux/2c0779fcb22e512cbdc613eb78200571
 # Migration to a newer database version is a risky operation to your data, so prepare and test the backup of the `postgres_data` volume.
 # NOTE: Ignored if `db` is not used.
-# DEFAULT: 13-3.1-alpine
-POSTGIS_IMAGE_VERSION=13-3.1-alpine
+# DEFAULT: 17-3.5-alpine
+POSTGIS_IMAGE_VERSION=17-3.5-alpine
 
 # Local admin username configuration for minio storage in local and standalone instances.
 # NOTE: Ignored if `minio` is not used.
@@ -520,9 +535,9 @@ DEBUG_WORKER_WRAPPER_DEBUGPY_PORT=5679
 # DEFAULT: ""
 DEBUG_QGIS_DEBUGPY_PORT=""
 
-# Host path which will be mounted by the `worker_wrapper` into the `worker` containers to facilitate development and debugging pythons files.
-# Will mount `qfc_worker`, `entrypoint.py`, and if existent - `qfieldcloud-sdk` and `libqfieldsync`.
-# If empty value or invalid value, the copied by docker or pip installed versions will be used.
+# Host path which will be mounted by the `worker_wrapper` into the `worker` containers to facilitate development and debugging python files.
+# Will mount `qfc_worker`, `entrypoint.py`, and if exists - `qfieldcloud-sdk` and `libqfieldsync`.
+# If empty or invalid value, the original code from the docker image (either copied or pip installed) will be used.
 # DEFAULT: ""
 DEBUG_QGIS_WORKER_HOST_PATH=""
 

.github/workflows/build_and_push.yml

@@ -1,6 +1,7 @@
 name: Build and Push
 on:
   push:
+  pull_request:
   release:
     types: [published]
 
@@ -12,61 +13,53 @@ jobs:
       fail-fast: true
       matrix:
         services:
-          -
-            service_name: app
+          - service_name: app
             dockerfile: docker-app/Dockerfile
             docker_context: docker-app
             docker_target: webserver_runtime
             enable_dockerhub: true
 
-          -
-            service_name: app-test
+          - service_name: app-test
             dockerfile: docker-app/Dockerfile
             docker_context: docker-app
             docker_target: webserver_test
             enable_dockerhub: false
 
-          -
-            service_name: worker-wrapper
+          - service_name: worker-wrapper
             dockerfile: docker-app/Dockerfile
             docker_context: docker-app
             docker_target: worker_wrapper_runtime
             enable_dockerhub: true
 
-          -
-            service_name: qgis
+          - service_name: qgis
             dockerfile: docker-qgis/Dockerfile
             docker_context: docker-qgis
             enable_dockerhub: true
 
-          -
-            service_name: nginx
+          - service_name: nginx
             dockerfile: docker-nginx/Dockerfile
             docker_context: docker-nginx
             enable_dockerhub: true
 
-          -
-            service_name: createbuckets
+          - service_name: createbuckets
             dockerfile: docker-createbuckets/Dockerfile
             docker_context: docker-createbuckets
             enable_dockerhub: true
 
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v4
 
-      -
-        name: Docker Metadata
+      - name: Docker Metadata
         id: docker_metadata
         uses: docker/metadata-action@v5
         with:
           images: |
-            name=ghcr.io/opengisch/qfieldcloud-${{ matrix.services.service_name }},enable=true
-            name=opengisch/qfieldcloud-${{ matrix.services.service_name }},enable=${{ matrix.services.enable_dockerhub && (github.event_name == 'release' || github.ref_name == 'master') }}
+            name=ghcr.io/${{ github.repository_owner }}/qfieldcloud-${{ matrix.services.service_name }},enable=true
+            name=${{ secrets.DOCKER_HUB_USERNAME }}/qfieldcloud-${{ matrix.services.service_name }},enable=${{ matrix.services.enable_dockerhub && (github.event_name == 'release' || github.ref_name == 'master' || (github.event_name == 'pull_request' && github.base_ref == 'master')) }}
 
           flavor: |
-            latest=auto
+            latest=${{ github.event_name == 'release' || github.ref_name == 'master' || (github.event_name == 'pull_request' && github.base_ref == 'master') }}
             prefix=,onlatest=false
             suffix=
 
@@ -76,36 +69,33 @@ jobs:
             type=match,pattern=v(.*),group=1
             type=edge,branch=master
 
-      -
-        name: Login to GitHub Container Repository (ghcr.io)
+      - name: Login to GitHub Container Repository (ghcr.io)
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      -
-        name: Login to Docker Hub
+      - name: Login to Docker Hub
         if: ${{ matrix.services.enable_dockerhub && (github.event_name == 'release' || github.ref_name == 'master') }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
-      -
-        name: Set up Docker Buildx
+      - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
 
-      -
-        name: Docker Build ${{ matrix.services.service_name }}
+      - name: Docker Build ${{ matrix.services.service_name }}
         id: docker_test_application
         uses: docker/build-push-action@v6
         with:
           file: ${{ matrix.services.dockerfile }}
           context: ${{ matrix.services.docker_context }}
+          platforms: linux/amd64,linux/arm64/v8
           push: ${{ matrix.services.enable_dockerhub && (github.event_name == 'release' || github.ref_name == 'master') }}
           target: ${{ matrix.services.docker_target }}
           tags: ${{ steps.docker_metadata.outputs.tags }}
-          cache-from: type=registry,ref=ghcr.io/opengisch/qfieldcloud-${{ matrix.services.service_name }}:buildcache
-          cache-to: type=registry,ref=ghcr.io/opengisch/qfieldcloud-${{ matrix.services.service_name }}:buildcache,mode=max
+          cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/qfieldcloud-${{ matrix.services.service_name }}:buildcache
+          cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/qfieldcloud-${{ matrix.services.service_name }}:buildcache,mode=max

.github/workflows/sync_translations.yml

@@ -1,50 +1,60 @@
 name: 🌎 Synchronize translations with Transifex
 
 on:
-    schedule:
-      - cron: "0 2 * * *"
-    workflow_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
+  workflow_dispatch:
 
 jobs:
-
   sync_translations:
-
     name: Synchronize Transifex translations
     runs-on: ubuntu-latest
 
     env:
       TX_TOKEN: ${{ secrets.TX_TOKEN }}
 
     steps:
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.NYUKI_TOKEN }}
 
-      - name: Install Transifex CLI
+      - name: Install transifex client
         run: |
+          # Detect architecture and download appropriate binary
+          ARCH=$(uname -m)
+          if [ "$ARCH" = "x86_64" ]; then
             curl -OL https://github.com/transifex/cli/releases/download/v1.6.17/tx-linux-amd64.tar.gz
             tar -xvzf tx-linux-amd64.tar.gz
+          elif [ "$ARCH" = "aarch64" ]; then
+            curl -OL https://github.com/transifex/cli/releases/download/v1.6.17/tx-linux-arm64.tar.gz
+            tar -xvzf tx-linux-arm64.tar.gz
+          else
+            echo "Unsupported architecture: $ARCH" && exit 1
+          fi
+          sudo mv tx /usr/local/bin/tx
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Copy .env file
         run: |
           cp .env.example .env
 
       - name: Perform docker pre operations
         run: |
-            docker compose pull
-            docker compose build
+          docker compose pull
+          docker compose build
 
       - name: Generate Django translation po files
         run: |
-            docker compose run --user root app python manage.py makemessages -l es
+          docker compose run --user root app python manage.py makemessages -l es
 
       - name: Push translation files to Transifex
-        run: ./tx push --source
+        run: tx push --source
 
       - name: Pull from Transifex
-        run: ./tx pull --all --minimum-perc 0 --force
+        run: tx pull --all --minimum-perc 0 --force
 
       - name: Add and commit new translations
         uses: EndBug/add-and-commit@v9

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 .coverage
 .vscode
 .env
+.env.test
 docker-compose.override.yml
 client/projects
 conf/mkcert/*
@@ -16,6 +17,7 @@ Pipfile*
 **/site-packages
 docker-qgis/libqfieldsync
 docker-qgis/qfieldcloud-sdk-python
+.venv
 
 # compiled translations
 docker-app/qfieldcloud/locale/**/*.mo

README.md

@@ -274,7 +274,7 @@ Do not forget to copy the site packages every time any of the `requirements*.txt
 
 QFieldCloud will automatically generate a certificate and its root certificate in `./conf/nginx/certs`.
 However, you need to trust the root certificate first,
-so other programs (e.g. curl) can create secure connection to the local QFieldCloud instance.
+so other programs (e.g. `curl`) can create secure connection to the local QFieldCloud instance.
 
 On Debian/Ubuntu, copy the root certificate to the directory with trusted certificates. Note the extension has been changed to `.crt`:
 
@@ -307,7 +307,7 @@ Now connecting with `curl` should fail with a similar error:
 
 ## Code style
 
-Code style done with [`precommit`](https://pre-commit.com):
+Code style done with [`pre-commit`](https://pre-commit.com):
 
     pip install pre-commit
     # install pre-commit hook
@@ -402,7 +402,7 @@ For great `nginx` logs, use:
 
 ### Storage
 
-You can use either the integrated `minio` object storage, or use an external provider (e. g. S3) with versioning enabled. Check the corresponding `STORAGE_*` environment variables for more info.
+You can use either the integrated `minio` object storage, or use an external provider (e.g. S3) with versioning enabled. Check the corresponding `STORAGE_*` environment variables for more info.
 
 ### Database
 

docker-app/Dockerfile

@@ -3,6 +3,7 @@
 ##########################
 
 # pull a builder image, the same as the base
+# This base image supports both AMD64 and ARM64 architectures
 FROM python:3.10-slim-bookworm AS build
 
 # Disable annoying pip version check, we don't care if pip is slightly older
@@ -11,17 +12,33 @@ ARG PIP_DISABLE_PIP_VERSION_CHECK=1
 # Do not create and use redundant cache dir in the current user home
 ARG PIP_NO_CACHE_DIR=1
 
-# install psycopg2 requirements
+# install psycopg2 and GDAL requirements for building packages like fiona
 RUN apt-get update \
     && apt-get install -y \
     libpq-dev \
     python3-dev \
     gcc \
+    g++ \
+# GDAL development headers needed for building fiona and other geospatial packages
+    libgdal-dev \
+    gdal-bin \
+    git \
+# GDAL development headers needed for building fiona and other geospatial packages on ARM64
+    g++ \
+# GDAL development headers needed for building fiona and other geospatial packages
+    libgdal-dev \
+    gdal-bin \
     && rm -rf /var/lib/apt/lists/*
 
 # install `pip-compile` (as part of `pip-tools`)
 RUN pip3 install pip-tools
 
+# Set GDAL environment variables for building packages like fiona
+ENV GDAL_CONFIG=/usr/bin/gdal-config
+
+# Set GDAL environment variables for building packages like fiona
+ENV GDAL_CONFIG=/usr/bin/gdal-config
+
 # install pip dependencies
 COPY ./requirements/requirements.txt /requirements/requirements.txt
 RUN pip3 install -r requirements/requirements.txt \
@@ -35,6 +52,7 @@ RUN ls -Q /usr/local/lib/python3.10/site-packages/botocore/data | grep -xv "endp
 ##########################
 
 # pull official base image
+# This base image supports both AMD64 and ARM64 architectures
 FROM python:3.10-slim-bookworm AS base
 
 # set work directory
@@ -47,12 +65,24 @@ ENV PYTHONUNBUFFERED=1
 # NOTE while using ARG would be more appropriate, the following vars would have to be redifined for each build stage.
 ENV PIP_DISABLE_PIP_VERSION_CHECK=1
 ENV PIP_NO_CACHE_DIR=1
+# GDAL configuration for building geospatial packages
+ENV GDAL_CONFIG=/usr/bin/gdal-config
+# GDAL configuration for building geospatial packages
+ENV GDAL_CONFIG=/usr/bin/gdal-config
 
 # install dependencies
 RUN apt-get update \
     && apt-get install -y \
 # GeoDjango as recommended at https://docs.djangoproject.com/en/4.1/ref/contrib/gis/install/geolibs/#installing-geospatial-libraries
     binutils libproj-dev gdal-bin \
+# GDAL development headers needed for building packages like fiona in test environments
+    libgdal-dev \
+# Build tools needed for compiling Python packages with C++ extensions
+    gcc g++ python3-dev \
+# GDAL development headers needed for building packages like fiona in test environments
+    libgdal-dev \
+# Build tools needed for compiling Python packages with C++ extensions
+    gcc g++ python3-dev \
 # needed for Django's `manage.py makemessages`
     gettext \
 # for development purposes only (optional dependency for `django-extensions`)