WIP: projects support

config/settings/base.py

@@ -92,6 +92,8 @@
     "treebeard",
     "corsheaders",
     "rules.apps.AutodiscoverRulesConfig",
+    "taggit",
+    "django_filters",
 ]
 
 LOCAL_APPS = [
@@ -101,6 +103,8 @@
     "metadata_catalogue.datasets.csw",
     "metadata_catalogue.datasets.geoapi",
     "metadata_catalogue.maps",
+    "metadata_catalogue.projects",
+    "metadata_catalogue.nina",
 ]
 # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
@@ -345,3 +349,7 @@
 
 CORS_ALLOWED_ORIGINS = env.list("DJANGO_CORS_ALLOWED_ORIGINS", default=[])
 CORS_ALLOW_CREDENTIALS = True
+
+
+PROJECTS_PROJECT_MODEL = "nina.Project"
+PROJECTS_PROJECTMEMBERSHIP_MODEL = "nina.ProjectMembership"

config/urls.py

@@ -23,6 +23,7 @@
     path("geoapi/", include("metadata_catalogue.datasets.geoapi.urls", namespace="geoapi")),
     path("datasets/", include("metadata_catalogue.datasets.urls")),
     path("api/", api.urls),
+    path("", include("metadata_catalogue.nina.urls")),
 ]
 
 

docker-compose.yml

@@ -163,6 +163,8 @@ services:
       - prod
     volumes:
       - statics:/statics
+    ports:
+      - 8000:80
 
   nginx-dev:
     hostname: nginx
@@ -172,9 +174,15 @@ services:
       context: ./nginx
     volumes:
       - ./media:/media_files
+    ports:
+      - 8000:80
 
   varnish:
     build:
       context: ./varnish
+
+
+  prosjektapi:
+    image: registry.gitlab.com/nina-data/prosjekt-oversikt/main/prosjektapi
     ports:
-      - 8000:80
+      - 8040:80

metadata_catalogue/core/management/commands/setup.py

@@ -25,5 +25,17 @@ def handle(self, *args, **options):
         if Language.objects.all().first() is None:
             call_command("loaddata", "languages_data.json.gz")
 
-        ServiceInfo = apps.get_model("datasets", "ServiceInfo")
-        ServiceInfo.get_solo()
+        try:
+            Topic = apps.get_model("nina", "Topic")
+            if Topic.objects.all().first() is None:
+                call_command("loaddata", "topics.json")
+        except:
+            print(traceback.format_exc())
+            print("Topics not loaded")
+
+        try:
+            ServiceInfo = apps.get_model("datasets", "ServiceInfo")
+            ServiceInfo.get_solo()
+        except:
+            print(traceback.format_exc())
+            print("Service info not created")

metadata_catalogue/core/templatetags/pagination.py

@@ -0,0 +1,24 @@
+from django import template
+
+register = template.Library()
+
+
+@register.simple_tag
+def query_transform(request, **kwargs):
+    updated = request.GET.copy()
+    for k, v in kwargs.items():
+        if v is not None:
+            updated[k] = v
+        else:
+            updated.pop(k, 0)
+
+    return updated.urlencode()
+
+
+@register.simple_tag
+def page_result(page_object, **kwargs):
+    first = ((page_object.number - 1) * page_object.paginator.per_page) + 1
+    first = first if page_object.paginator.count > 0 else 0
+    second = (page_object.number * page_object.paginator.per_page) + 1
+    second = second if second < page_object.paginator.count else page_object.paginator.count
+    return f"{first}-{second} of {page_object.paginator.count} results"

metadata_catalogue/fixtures/topics.json

@@ -0,0 +1 @@
+[{"model": "nina.topic", "pk": 1, "fields": {"name": "Polarområdene\n"}}, {"model": "nina.topic", "pk": 2, "fields": {"name": "Kyst/marint/hav\n"}}, {"model": "nina.topic", "pk": 3, "fields": {"name": "Ferskvann\n"}}, {"model": "nina.topic", "pk": 4, "fields": {"name": "Fjell\n"}}, {"model": "nina.topic", "pk": 5, "fields": {"name": "Skog\n"}}, {"model": "nina.topic", "pk": 6, "fields": {"name": "Våtmark/myr\n"}}, {"model": "nina.topic", "pk": 7, "fields": {"name": "Åpent lavland og Kulturlandskap\n"}}, {"model": "nina.topic", "pk": 8, "fields": {"name": "Natur i by\n"}}, {"model": "nina.topic", "pk": 9, "fields": {"name": "Laksefisk\n"}}, {"model": "nina.topic", "pk": 10, "fields": {"name": "Botanikk/vegetasjon\n"}}, {"model": "nina.topic", "pk": 11, "fields": {"name": "Insekter\n"}}, {"model": "nina.topic", "pk": 12, "fields": {"name": "Sjøfugl\n"}}, {"model": "nina.topic", "pk": 13, "fields": {"name": "Rovvilt/rovdyr\n"}}, {"model": "nina.topic", "pk": 14, "fields": {"name": "Hjortevilt\n"}}, {"model": "nina.topic", "pk": 15, "fields": {"name": "Trua arter\n"}}, {"model": "nina.topic", "pk": 16, "fields": {"name": "Fremmede arter\n"}}, {"model": "nina.topic", "pk": 17, "fields": {"name": "Klima/klimaendring\n"}}, {"model": "nina.topic", "pk": 18, "fields": {"name": "Forurensning\n"}}, {"model": "nina.topic", "pk": 19, "fields": {"name": "Fornybar energi\n"}}, {"model": "nina.topic", "pk": 20, "fields": {"name": "Havbruk\n"}}, {"model": "nina.topic", "pk": 21, "fields": {"name": "Samferdsel/infrastruktur\n"}}, {"model": "nina.topic", "pk": 22, "fields": {"name": "Jakt/fiske/fiskeri\n"}}, {"model": "nina.topic", "pk": 23, "fields": {"name": "Friluftsliv/Turisme\n"}}, {"model": "nina.topic", "pk": 24, "fields": {"name": "Arealbruk/arealendring/areal\n"}}, {"model": "nina.topic", "pk": 25, "fields": {"name": "Overvåking/Indikatorer\n"}}, {"model": "nina.topic", "pk": 26, "fields": {"name": "Telemetri\n"}}, {"model": "nina.topic", "pk": 27, "fields": {"name": "GIS/Fjernmåling\n"}}, {"model": "nina.topic", "pk": 28, "fields": {"name": "Vannforskriften\n"}}, {"model": "nina.topic", "pk": 29, "fields": {"name": "Økologisk tilstand/Naturindeks\n"}}, {"model": "nina.topic", "pk": 30, "fields": {"name": "Kartlegging naturtyper/NiN\n"}}, {"model": "nina.topic", "pk": 31, "fields": {"name": "Genetikk/eDNA, miljø-DNA\n"}}, {"model": "nina.topic", "pk": 32, "fields": {"name": "Økosystemtjenester/Naturgoder\n"}}, {"model": "nina.topic", "pk": 33, "fields": {"name": "Restaurering\n"}}, {"model": "nina.topic", "pk": 34, "fields": {"name": "Vern/nasjonalparker/forvaltning\n"}}, {"model": "nina.topic", "pk": 35, "fields": {"name": "Konsekvensutredning\n"}}, {"model": "nina.topic", "pk": 36, "fields": {"name": "Samfunn/menneske\n"}}, {"model": "nina.topic", "pk": 37, "fields": {"name": "Folkeforskning/citizen science\n"}}, {"model": "nina.topic", "pk": 38, "fields": {"name": "Bistand\n"}}]

metadata_catalogue/maps/api.py

@@ -1,12 +1,10 @@
 import uuid
 from typing import List
 
-from django.db.models import Q
 from ninja import Router
 from ninja.responses import codes_4xx
 
 from . import models, schema
-from .enums import Visibility
 
 maps_router = Router()
 
@@ -77,13 +75,6 @@ def get_portal_maps(request, portal_uuid: uuid.UUID):
         portal = models.Portal.objects.get(uuid=portal_uuid)
         if not request.user.has_perm("maps.portal_view", portal):
             return 404, {"message": "Not found"}
-
-        expression = Q()
-        if request.user.is_authenticated:
-            if not request.user.is_staff:
-                expression = Q(map__visibility=Visibility.PUBLIC) | Q(map__owner=request.user)
-        else:
-            expression = Q(map__visibility=Visibility.PUBLIC)
-        return 200, portal.maps.filter(expression).select_related("map")
+        return 200, portal.get_visible_maps(request=request)
     except models.Portal.DoesNotExist:
         return 404, {"message": "Not found"}

metadata_catalogue/maps/conf.py

@@ -4,3 +4,4 @@
 
 class MapsConf(AppConf):
     API_PREFIX = "api-1.0.0"
+    CUSTOM_RULES = False

metadata_catalogue/maps/models.py

@@ -2,6 +2,7 @@
 
 from django.conf import settings
 from django.db import models
+from django.http import HttpRequest
 from django.urls import reverse
 from polymorphic.models import PolymorphicModel
 from slugify import slugify
@@ -258,6 +259,15 @@ class Meta:
     def __str__(self) -> str:
         return self.title
 
+    def get_visible_maps(self, request: HttpRequest):
+        expression = models.Q()
+        if request.user.is_authenticated:
+            if not request.user.is_staff:
+                expression = models.Q(map__visibility=Visibility.PUBLIC) | models.Q(map__owner=request.user)
+        else:
+            expression = models.Q(map__visibility=Visibility.PUBLIC)
+        return self.maps.filter(expression).select_related("map")
+
 
 class PortalMap(models.Model):
     map = models.ForeignKey("maps.Map", on_delete=models.CASCADE, related_name="portals")

metadata_catalogue/maps/rules.py

@@ -1,6 +1,7 @@
 import rules
 
 from . import enums
+from .conf import settings
 
 
 @rules.predicate
@@ -13,13 +14,13 @@ def is_public(user, object):
     return object.visibility == enums.Visibility.PUBLIC
 
 
-rules.add_perm("maps.map_view", is_public | is_owner | rules.is_staff)
-rules.add_perm("maps.map_edit", is_owner | rules.is_staff)
-rules.add_perm("maps.map_add", is_owner | rules.is_staff)
-rules.add_perm("maps.map_delete", is_owner | rules.is_staff)
+if not settings.MAPS_CUSTOM_RULES:
+    rules.add_perm("maps.map_view", is_public | is_owner | rules.is_staff)
+    rules.add_perm("maps.map_edit", is_owner | rules.is_staff)
+    rules.add_perm("maps.map_add", is_owner | rules.is_staff)
+    rules.add_perm("maps.map_delete", is_owner | rules.is_staff)
 
-
-rules.add_perm("maps.portal_view", is_public | is_owner | rules.is_staff)
-rules.add_perm("maps.portal_edit", is_owner | rules.is_staff)
-rules.add_perm("maps.portal_add", is_owner | rules.is_staff)
-rules.add_perm("maps.portal_delete", is_owner | rules.is_staff)
+    rules.add_perm("maps.portal_view", is_public | is_owner | rules.is_staff)
+    rules.add_perm("maps.portal_edit", is_owner | rules.is_staff)
+    rules.add_perm("maps.portal_add", is_owner | rules.is_staff)
+    rules.add_perm("maps.portal_delete", is_owner | rules.is_staff)

metadata_catalogue/nina/admin.py

@@ -0,0 +1,23 @@
+from django.contrib.admin import ModelAdmin, register
+
+from .models import Category, Department, Project, Topic
+
+
+@register(Project)
+class ProjectAdmin(ModelAdmin):
+    search_fields = ["name", "slug", "description"]
+
+
+@register(Department)
+class DepartmentAdmin(ModelAdmin):
+    search_fields = ["name", "slug", "description"]
+
+
+@register(Category)
+class CategoryAdmin(ModelAdmin):
+    search_fields = ["name"]
+
+
+@register(Topic)
+class TopicAdmin(ModelAdmin):
+    search_fields = ["name"]

metadata_catalogue/nina/conf.py

@@ -0,0 +1,6 @@
+from appconf import AppConf
+from django.conf import settings
+
+
+class NINAConf(AppConf):
+    pass

metadata_catalogue/nina/filters.py

@@ -0,0 +1,22 @@
+import django_filters
+from taggit.forms import TagField
+
+from .forms import ProjectSearchForm
+from .models import Project
+
+
+class TagFilter(django_filters.CharFilter):
+    field_class = TagField
+
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault("lookup_expr", "in")
+        super().__init__(*args, **kwargs)
+
+
+class ProjectFilter(django_filters.FilterSet):
+    tags = TagFilter(field_name="tags__name")
+
+    class Meta:
+        model = Project
+        form = ProjectSearchForm
+        fields = ["status", "topics", "category", "departments", "customer"]

metadata_catalogue/nina/forms.py

@@ -0,0 +1,35 @@
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import Submit
+from django import forms
+
+from .models import Project
+
+
+class ProjectSearchForm(forms.ModelForm):
+    class Meta:
+        model = Project
+        fields = ["status", "topics", "category", "departments", "customer"]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.helper = FormHelper()
+        self.helper.form_method = "get"
+
+        self.helper.add_input(Submit("submit", "Search"))
+
+
+class ProjectEditForm(forms.ModelForm):
+    class Meta:
+        model = Project
+        fields = [
+            "description",
+            "tags",
+            "topics",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.helper = FormHelper()
+        self.helper.form_method = "post"
+
+        self.helper.add_input(Submit("submit", "Edit"))

metadata_catalogue/nina/libs/harvesters.py

@@ -0,0 +1,88 @@
+from typing import Dict
+
+import requests
+from django.contrib.auth import get_user_model
+
+from metadata_catalogue.datasets.models import Organization
+
+from ..models import Category, Department, Project
+
+User = get_user_model()
+
+
+def _fetch_paginated_project(url: str, limit=50):
+    offset = 0
+    found = None
+
+    while found is None or found == limit:
+        query = f"{url}?rows={limit}&start={offset}"
+        response = requests.get(query)
+        if response.status_code == 200:
+            data = response.json()
+
+            results = data.get("result").get("results")
+            if results:
+                yield results
+            else:
+                break
+
+            offset += limit
+            found = data.get("result").get("count")
+        else:
+            response.raise_for_status()
+
+
+def _process_project(project: dict):
+    p, created = Project.objects.get_or_create(
+        id=project.get("id"),
+        defaults={
+            "name": project.get("title"),
+            "description": project.get("notes"),
+            "slug": f'prj-{project.get("id")}',
+        },
+    )
+
+    p.status = project.get("project_state")
+    p.budget = project.get("budget")
+    p.start_date = project.get("startdate")
+    p.end_date = project.get("enddate")
+
+    p.category, _ = Category.objects.get_or_create(name=project.get("category"))
+    p.customer, _ = Organization.objects.get_or_create(name=project.get("customer"))
+
+    for group in project.get("groups"):
+        d, created = Department.objects.get_or_create(
+            id=group.get("id"),
+            defaults={
+                "name": group.get("title"),
+                "description": group.get("description"),
+                "slug": f'dpt-{project.get("id")}',
+            },
+        )
+
+        p.departments.add(d)
+
+    p.save()
+
+    if project.get("maintainer_email"):
+        u, created = User.objects.get_or_create(email=project.get("maintainer_email"))
+        if created:
+            u.set_unusable_password()
+            u.save()
+
+        p.members.add(u)
+
+
+def prosjektoversikt(url: str, limit=50):
+    """
+    Harvest projects and departments from prosjekt oversikt APIs
+    NOTE: API scheme resembles the CKAN APIs
+
+    Attributes:
+        url (str): URL of the Prosjekt Oversikt API, it should point to the base url (just before the `/api/`) without trailing /
+            example: https://prosjekt-oversikt.nina.no/ckan
+    """
+
+    for projects in _fetch_paginated_project(f"{url}/api/3/action/package_search", limit=limit):
+        for project in projects:
+            _process_project(project)

metadata_catalogue/nina/management/commands/harvest_prosjektoversikt.py

@@ -0,0 +1,14 @@
+from django.core.management.base import BaseCommand
+
+from ...libs.harvesters import prosjektoversikt
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("url", type=str, help="projects api endpoint")
+        parser.add_argument("--limit", type=int, help="how many lines to read at each request", default=50)
+
+    def handle(self, *args, **options):
+        url = options.get("url")
+        limit = options.get("limit")
+        prosjektoversikt(url, limit=limit)