Merge pull request #57 from EOSC-Data-Commons/include-harvester-image

Add harvester to docker-compose

Author: jos-no

docker-compose.yml

@@ -225,6 +225,17 @@ services:
         networks:
             - warehouse-backend
 
+    # https://github.com/EOSC-Data-Commons/metadata-crawlers/pkgs/container/metadata-crawlers
+    harvester:
+        image: ghcr.io/eosc-data-commons/metadata-crawlers:latest
+        env_file:
+            - .env
+        volumes:
+            - harvester-logs:/app/logs
+        entrypoint: ["python", "-m", "harvester"]
+        networks:
+            - warehouse-backend
+
 volumes:
     postgres-data:
     pgadmin-data:
@@ -234,6 +245,7 @@ volumes:
     flower-data:
     search-api-data:
     frontend-data:
+    harvester-logs:
 
 networks:
     warehouse-backend:

env.template

@@ -11,3 +11,5 @@ EMBEDDING_MODEL=BAAI/bge-small-en-v1.5 # https://qdrant.github.io/fastembed/exam
 EMBEDDING_DIMS=384 # depends on EMBEDDING_MODEL
 INDEX_NAME=test_datacite
 CELERY_BATCH_SIZE=250
+ENVIRONMENT= # production/staging/dev/local
+WAREHOUSE_API_URL=