Datasets UI

.pre-commit-config.yaml

@@ -21,30 +21,32 @@ repos:
     rev: '1.15.0'
     hooks:
       - id: django-upgrade
-        args: ['--target-version', '4.2']
+        args: ['--target-version', '5.0']
 
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.15.0
     hooks:
       - id: pyupgrade
         args: [--py311-plus]
 
-  - repo: https://github.com/psf/black
-    rev: 23.10.1
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.5
     hooks:
-      - id: black
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
+      - id: ruff
+      - id: ruff-format
 
   - repo: https://github.com/Riverside-Healthcare/djLint
     rev: v1.34.0
     hooks:
       - id: djlint-reformat-django
       - id: djlint-django
 
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.4.0
+    hooks:
+      - id: detect-secrets
+        args: [--exclude-files=metadata_catalogue/fixtures/users.json, --exclude-files=config/settings/test.py, --exclude-files=config/settings/local.py]
+
 # sets up .pre-commit-ci.yaml to ensure pre-commit dependencies stay up to date
 ci:
   autoupdate_schedule: weekly

Dockerfile

@@ -1,30 +1,34 @@
-FROM registry.opensuse.org/opensuse/leap:latest as base
+FROM registry.opensuse.org/opensuse/leap:latest AS base
 RUN --mount=type=cache,target=/var/cache/zypper \
     zypper install --no-recommends -y python311 gdal proj
 WORKDIR /app
 ENV PYTHONPATH=/app/.venv/lib
 ENV PATH=$PATH:/app/.venv/bin
 
-FROM scratch as source
+FROM scratch AS source
 WORKDIR /app
 COPY config config
 COPY metadata_catalogue metadata_catalogue
 COPY manage.py .
 
-FROM base as pdm
+FROM base AS pdm
 RUN --mount=type=cache,target=/var/cache/zypper \
     zypper install --no-recommends -y python311-pip git
 RUN python3.11 -m pip install pdm
 COPY ./pyproject.toml ./pdm.lock .
 
-FROM pdm as production
+FROM pdm AS pdm-node
+RUN --mount=type=cache,target=/var/cache/zypper \
+    zypper install --no-recommends -y nodejs20 npm20
+
+FROM pdm AS production
 RUN --mount=type=cache,target=/var/cache/zypper \
     zypper install --no-recommends -y gdal-devel gcc gcc-c++ python311-devel
 RUN pdm add gdal==$(rpm -q --queryformat='%{VERSION}' gdal)
 RUN --mount=type=cache,target=/root/.cache/pdm \
     pdm install -G production
 
-FROM pdm as translation
+FROM pdm AS translation
 RUN --mount=type=cache,target=/var/cache/zypper \
     zypper install --no-recommends -y gettext-tools
 COPY --from=production /app/.venv .venv
@@ -34,15 +38,27 @@ RUN DATABASE_URL="" DJANGO_BASE_SCHEMA_URL="" \
   DJANGO_SETTINGS_MODULE="config.settings.test" \
   pdm run ./manage.py compilemessages
 
-FROM base as django
+FROM pdm-node AS tailwind
+COPY --from=production /app .
+COPY --from=translation /app/locale locale
+COPY --from=source /app .
+RUN DATABASE_URL="" \
+DJANGO_SETTINGS_MODULE="config.settings.test" \
+python manage.py tailwind install
+RUN DATABASE_URL="" \
+DJANGO_SETTINGS_MODULE="config.settings.test" \
+python manage.py tailwind build
+
+FROM base AS django
 COPY --from=production /app .
 COPY --from=translation /app/locale locale
+COPY --from=tailwind /app/metadata_catalogue/theme/static /app/metadata_catalogue/theme/static
 COPY --from=source /app .
 RUN mkdir media
 COPY entrypoint.sh .
 ENTRYPOINT ["./entrypoint.sh"]
 
-FROM pdm as dev
+FROM pdm-node AS dev
 COPY --from=django /app/.venv .venv
 RUN --mount=type=cache,target=/root/.cache/pdm \
     pdm install --dev

bruno/Dockerfile

@@ -1,8 +1,8 @@
-FROM node:18-bookworm as base
+FROM node:18-bookworm AS base
 RUN npm install -g @usebruno/cli
 WORKDIR /app
 
-FROM base as deps
+FROM base AS deps
 COPY ./bruno/catalogue/package.json .
 RUN npm install
 

config/settings/base.py

@@ -1,6 +1,7 @@
 """
 Base settings to build other settings files upon.
 """
+
 from pathlib import Path
 
 import environ
@@ -49,6 +50,8 @@
 DATABASES["default"]["ENGINE"] = "psqlextra.backend"
 POSTGRES_EXTRA_DB_BACKEND_BASE = "django.contrib.gis.db.backends.postgis"
 DATABASES["default"]["ATOMIC_REQUESTS"] = True
+DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=None)
+
 # https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-DEFAULT_AUTO_FIELD
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
@@ -76,14 +79,16 @@
 
 THIRD_PARTY_APPS = [
     "crispy_forms",
-    "crispy_bootstrap5",
+    "crispy_tailwind",
+    "allauth_ui",
     "allauth",
     "allauth.account",
+    "allauth.socialaccount",
+    "allauth.socialaccount.providers.openid_connect",
     "django_probes",
     "health_check",
     "health_check.db",
     "health_check.contrib.migrations",
-    "django_q",
     "countries_plus",
     "languages_plus",
     "psqlextra",
@@ -97,6 +102,13 @@
     "drf_spectacular",
     "drf_standardized_errors",
     "django_filters",
+    "tailwind",
+    "django_tables2",
+    "widget_tweaks",
+    "slippers",
+    "fontawesomefree",
+    "leaflet",
+    "procrastinate.contrib.django",
 ]
 
 LOCAL_APPS = [
@@ -106,6 +118,7 @@
     "metadata_catalogue.datasets.csw",
     "metadata_catalogue.datasets.geoapi",
     "metadata_catalogue.maps",
+    "metadata_catalogue.theme",
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
@@ -126,7 +139,7 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#auth-user-model
 AUTH_USER_MODEL = "users.User"
 # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url
-LOGIN_REDIRECT_URL = "users:redirect"
+LOGIN_REDIRECT_URL = "home"
 # https://docs.djangoproject.com/en/dev/ref/settings/#login-url
 LOGIN_URL = "account_login"
 
@@ -142,7 +155,9 @@
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#auth-password-validators
 AUTH_PASSWORD_VALIDATORS = [
-    {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"
+    },
     {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
     {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
     {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
@@ -209,6 +224,7 @@
                 "django.contrib.messages.context_processors.messages",
                 "metadata_catalogue.users.context_processors.allauth_settings",
             ],
+            "builtins": ["slippers.templatetags.slippers"],
         },
     }
 ]
@@ -217,8 +233,8 @@
 FORM_RENDERER = "django.forms.renderers.TemplatesSetting"
 
 # http://django-crispy-forms.readthedocs.io/en/latest/install.html#template-packs
-CRISPY_TEMPLATE_PACK = "bootstrap5"
-CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap5"
+CRISPY_TEMPLATE_PACK = "tailwind"
+CRISPY_ALLOWED_TEMPLATE_PACKS = "tailwind"
 
 # FIXTURES
 # ------------------------------------------------------------------------------
@@ -305,25 +321,34 @@
 # https://django-allauth.readthedocs.io/en/latest/forms.html
 ACCOUNT_FORMS = {"signup": "metadata_catalogue.users.forms.UserSignupForm"}
 
-# DJANGO Q
-Q_CLUSTER = {
-    "name": "viltkamera",
-    "workers": env.int("Q_WORKERS", default=1),
-    "recycle": 500,
-    "retry": env.int("Q_RETRY", default=6000),
-    "timeout": env.int("Q_TIMEOUT", default=5999),
-    "cpu_affinity": env.int("Q_AFFINITY", default=0),
-    "save_limit": env.int("Q_SAVE_LIMIT", default=250),
-    "queue_limit": 500,
-    "label": "Django Q",
-    "orm": "default",
-}
-
-if Q_BROKER_CLASS := env(
-    "Q_BROKER_CLASS",
-    default="metadata_catalogue.core.brokers.GeneralPurposeBroker",
-):
-    Q_CLUSTER.update({"broker_class": Q_BROKER_CLASS})
+ALLAUTH_UI_THEME = "nina"
+
+if env("SOCIALACCOUNT_ADAPTER", default=None):
+    SOCIALACCOUNT_ADAPTER = env("SOCIALACCOUNT_ADAPTER")
+
+if OIDC_CLIENT_ID := env("OIDC_CLIENT_ID", default=None):
+    SOCIALACCOUNT_ONLY = True
+    SOCIALACCOUNT_STORE_TOKENS = env("SOCIALACCOUNT_STORE_TOKENS", default=False)
+    extra = {}
+    if OIDC_SECRET := env("OIDC_SECRET", default=None):
+        extra["secret"] = OIDC_SECRET
+
+    SOCIALACCOUNT_PROVIDERS = {
+        "openid_connect": {
+            "OAUTH_PKCE_ENABLED": env("OAUTH_PKCE_ENABLED", default=False),
+            "APPS": [
+                {
+                    "provider_id": env("OIDC_PROVIDER_ID"),
+                    "name": env("OIDC_PROVIDER_NAME"),
+                    "client_id": OIDC_CLIENT_ID,
+                    "settings": {
+                        "server_url": env("OIDC_PROVIDER_URL"),
+                    },
+                    **extra,
+                },
+            ],
+        },
+    }
 
 
 CSW = {
@@ -349,6 +374,7 @@
 GEOAPI_SETTINGS_CACHE_ENABLED = env.bool("GEOAPI_SETTINGS_CACHE_ENABLED", default=False)
 
 CORS_ALLOWED_ORIGINS = env.list("DJANGO_CORS_ALLOWED_ORIGINS", default=[])
+CORS_URLS_REGEX = r"^/api/.*$"
 CORS_ALLOW_CREDENTIALS = True
 
 MAPS_API_KEY = env("MAPS_API_KEY", default=None)
@@ -381,3 +407,8 @@
 }
 
 DRF_STANDARDIZED_ERRORS = {"ENABLE_IN_DEBUG_FOR_UNHANDLED_EXCEPTIONS": True}
+
+TAILWIND_APP_NAME = "metadata_catalogue.theme"
+
+IPT_SOURCES = env.list("IPT_SOURCES", default=[])
+IPT_SOURCES_CRON = env("IPT_SOURCES_CRON", default="0 0 * * 0")

config/settings/local.py

@@ -11,9 +11,13 @@
     default="iUb7AqZQ5PzH8EeOdXOQgwwvfhbtyQWuziGY8XfG6hAM1KVwH0Ukt8q4nAuKHF5m",
 )
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
-ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1", "django", "django-dev"] + env.list(
-    "DJANGO_ALLOWED_HOSTS", default=[""]
-)
+ALLOWED_HOSTS = [
+    "localhost",
+    "0.0.0.0",
+    "127.0.0.1",
+    "django",
+    "django-dev",
+] + env.list("DJANGO_ALLOWED_HOSTS", default=[""])
 
 # CACHES
 # ------------------------------------------------------------------------------
@@ -28,7 +32,9 @@
 # EMAIL
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
-EMAIL_BACKEND = env("DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend")
+EMAIL_BACKEND = env(
+    "DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.console.EmailBackend"
+)
 
 # django-debug-toolbar
 # ------------------------------------------------------------------------------
@@ -37,9 +43,16 @@
 # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#middleware
 MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"]  # noqa: F405
 # https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config
+
+
+def show_toolbar(request):
+    return True
+
+
 DEBUG_TOOLBAR_CONFIG = {
     "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
     "SHOW_TEMPLATE_CONTEXT": True,
+    "SHOW_TOOLBAR_CALLBACK": show_toolbar,
 }
 # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#internal-ips
 INTERNAL_IPS = ["127.0.0.1", "10.0.2.2"]
@@ -62,3 +75,5 @@
 
 
 INSTALLED_APPS += ["models2puml"]
+
+SOCIALACCOUNT_ADAPTER = "metadata_catalogue.users.adapters.SocialAccountAdapter"

config/settings/production.py

@@ -39,16 +39,23 @@
 SESSION_COOKIE_SECURE = env.bool("SESSION_COOKIE_SECURE", default=True)
 # https://docs.djangoproject.com/en/dev/ref/settings/#csrf-cookie-secure
 CSRF_COOKIE_SECURE = env.bool("CSRF_COOKIE_SECURE", default=True)
+
+CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[])
 # https://docs.djangoproject.com/en/dev/topics/security/#ssl-https
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-seconds
 # TODO: set this to 60 seconds first and then to 518400 once you prove the former works
 SECURE_HSTS_SECONDS = 60
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-include-subdomains
-SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool("DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True)
+SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool(
+    "DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True
+)
 # https://docs.djangoproject.com/en/dev/ref/settings/#secure-hsts-preload
 SECURE_HSTS_PRELOAD = env.bool("DJANGO_SECURE_HSTS_PRELOAD", default=True)
 # https://docs.djangoproject.com/en/dev/ref/middleware/#x-content-type-options-nosniff
-SECURE_CONTENT_TYPE_NOSNIFF = env.bool("DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True)
+SECURE_CONTENT_TYPE_NOSNIFF = env.bool(
+    "DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True
+)
+USE_X_FORWARDED_HOST = env.bool("DJANGO_USE_X_FORWARDED_HOST", default=False)
 
 # STATIC
 # ------------------------

config/settings/test.py

@@ -2,8 +2,12 @@
 With these settings, tests run faster.
 """
 
+import os
+
+os.environ["DJANGO_BASE_SCHEMA_URL"] = ""
+
 from .base import *  # noqa
-from .base import env
+from .base import env  # noqa: E402
 
 # GENERAL
 # ------------------------------------------------------------------------------
@@ -32,6 +36,6 @@
 # MEDIA
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#media-url
-MEDIA_URL = "http://media.testserver"
+MEDIA_URL = "http://media.testserver/"
 # Your stuff...
 # ------------------------------------------------------------------------------