videos: required transform rules / setup: separate rdm and videos

.github/workflows/tests.yml

@@ -14,34 +14,34 @@ on:
     branches: master
   schedule:
     # * is a special character in YAML so you have to quote this string
-    - cron:  '0 3 * * 6'
+    - cron: "0 3 * * 6"
   workflow_dispatch:
     inputs:
       reason:
-        description: 'Reason'
+        description: "Reason"
         required: false
-        default: 'Manual trigger'
+        default: "Manual trigger"
 
 jobs:
-  Tests:
+  RDMTests:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-          python-version: [3.9]
-          requirements-level: [pypi]
-          db-service: [postgresql14]
-          include:
+        python-version: [3.9]
+        requirements-level: [pypi]
+        db-service: [postgresql14]
+        include:
           - db-service: postgresql14
             DB_EXTRAS: "postgresql"
 
     env:
       DB: ${{ matrix.db-service }}
-      EXTRAS: tests
+      EXTRAS: rdm,tests
     steps:
       - name: Install python-ldap dependencies
         run: |
-            sudo apt-get update
-            sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -61,4 +61,43 @@ jobs:
           docker compose --version
 
       - name: Run tests
-        run: ./run-tests.sh
+        run: ./run-tests.sh rdm
+  VideosTests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.9]
+        requirements-level: [pypi]
+        db-service: [postgresql14]
+        include:
+          - db-service: postgresql14
+            DB_EXTRAS: "postgresql"
+
+    env:
+      DB: ${{ matrix.db-service }}
+      EXTRAS: videos,tests
+    steps:
+      - name: Install python-ldap dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: setup.cfg
+
+      - name: Install dependencies
+        run: |
+          pip install ".[$EXTRAS]"
+          pip freeze
+          docker --version
+          docker compose --version
+
+      - name: Run tests
+        run: ./run-tests.sh videos

README.rst

@@ -7,7 +7,81 @@
  cds-migrator-kit
 ==================
 
+Installation
+============
+
+Default Installation (without RDM or Videos)
+---------------------------------------------
+To install the package without RDM or videos, run:
+
+.. code-block:: bash
+
+    pip install .
+
+Installation for RDM
+----------------------
+To install the package with RDM, run:
+
+.. code-block:: bash
+
+    pip install ".[rdm]"
+
+To see available RDM commands, run:
+
+.. code-block:: bash
+
+    invenio migration --help
+
+Installation for Videos
+-----------------------
+To install the package with cds-videos, run:
+
+.. code-block:: bash
+
+    pip install ".[videos]"
+
+To see available videos commands, run:
+
+.. code-block:: bash
+
+    invenio migration videos --help
+
+Running Tests Locally
+=====================
+
+For RDM
+--------
+Install rdm and test dependencies:
+
+.. code-block:: bash
+
+    pip install ".[rdm,tests]"
+
+
+Run the tests with ignoring `cds-videos` tests:
+
+.. code-block:: bash
+
+    ./run-tests.sh rdm
+
+For Videos
+----------
+Install videos and test dependencies:
+
+.. code-block:: bash
+
+    pip install ".[videos,tests]"
+
+Run the video tests:
+
+.. code-block:: bash
+
+    ./run-tests.sh videos
+
+
 To run the interface:
-```
-gunicorn -b :8080 --timeout 120 --graceful-timeout 60 cds_migrator_kit.app:app
-```
+=====================
+.. code-block:: bash
+
+    gunicorn -b :8080 --timeout 120 --graceful-timeout 60 cds_migrator_kit.app:app
+

cds_migrator_kit/base_config.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# cds-migrator-kit is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+"""Migration configuration for CDS Migrator Kit."""
+
+from cds_migrator_kit.import_utils import import_module
+
+selected_config = None
+
+# Check for `rdm` dependencies
+if import_module("cds_rdm.__init__"):
+    from cds_migrator_kit.rdm import migration_config as selected_config
+
+# Check for `videos` dependencies
+elif import_module("cds.version"):
+    from cds_migrator_kit.videos import migration_config as selected_config
+
+# If no valid module is found, use default one
+if selected_config is None:
+    from cds_migrator_kit import config as selected_config
+
+# Set the selected config module
+globals().update(vars(selected_config))

cds_migrator_kit/base_minter.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# cds-migrator-kit is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+"""Minter configuration for CDS Migrator Kit."""
+
+import warnings
+
+from cds_migrator_kit.import_utils import import_module
+
+# Default: No minter
+selected_minter = None
+
+# Check if `rdm` is installed and set the minter
+if import_module("cds_rdm.__init__"):
+    from cds_rdm.minters import legacy_recid_minter as selected_minter
+else:
+    warnings.warn(
+        "No valid PID minter found. Ensure `rdm` is installed.", RuntimeWarning
+    )
+
+# Expose the minter function
+legacy = selected_minter

cds_migrator_kit/cli.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# cds-migrator-kit is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+"""cds-migrator-kit command line module."""
+
+import click
+
+from cds_migrator_kit.import_utils import import_module
+
+
+@click.group()
+def cli():
+    """Base CLI command that loads the subcommands."""
+    pass
+
+
+# Check for `rdm` dependencies
+if import_module("cds_rdm.__init__"):
+    from cds_migrator_kit.rdm.cli import migration
+    cli = migration
+
+# Check for `videos` dependencies
+if import_module("cds.version"):
+    from cds_migrator_kit.videos.weblecture_migration.cli import videos
+    cli.add_command(videos, "videos")

cds_migrator_kit/extract/__init__.py

@@ -2,7 +2,7 @@
 #
 # Copyright (C) 2025 CERN.
 #
-# CDS-Videos is free software; you can redistribute it and/or modify it under
+# cds-migrator-kit is free software; you can redistribute it and/or modify it under
 # the terms of the MIT License; see LICENSE file for more details.
 
 """Extract module."""

cds_migrator_kit/import_utils.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# cds-migrator-kit is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+"""Utility function for dynamically checking module availability."""
+
+import importlib
+
+
+def import_module(module_name):
+    """Try to import a module, return True if successful, otherwise False."""
+    try:
+        importlib.import_module(module_name)
+        return True
+    except ImportError:
+        return False

cds_migrator_kit/rdm/cli.py

@@ -19,7 +19,6 @@
 from cds_migrator_kit.rdm.records.streams import (  # UserStreamDefinition,
     RecordStreamDefinition,
 )
-from cds_migrator_kit.rdm.runner import Runner
 from cds_migrator_kit.rdm.stats.runner import RecordStatsRunner
 from cds_migrator_kit.rdm.stats.streams import RecordStatsStreamDefinition
 from cds_migrator_kit.rdm.users.runner import PeopleAuthorityRunner, SubmitterRunner
@@ -30,6 +29,7 @@
 from cds_migrator_kit.rdm.users.transform.xml_processing.models.people import (
     PeopleAuthority,
 )
+from cds_migrator_kit.runner.runner import Runner
 
 cli_logger = logging.getLogger("migrator")
 

cds_migrator_kit/rdm/migration_config.py

@@ -368,9 +368,6 @@ def _(x):  # needed to avoid start time failure with lazy strings
 CDS_MIGRATOR_KIT_LOGS_PATH = logs_dir
 
 CDS_MIGRATOR_KIT_STREAM_CONFIG = "cds_migrator_kit/rdm/streams.yaml"
-CDS_MIGRATOR_KIT_VIDEOS_STREAM_CONFIG = (
-    "cds_migrator_kit/videos/weblecture_migration/streams.yaml"
-)
 
 RDM_RECORDS_IDENTIFIERS_SCHEMES = {
     **RDM_RECORDS_IDENTIFIERS_SCHEMES,

cds_migrator_kit/runner/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# cds-migrator-kit is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""Runner module."""

cds_migrator_kit/rdm/runner.py → cds_migrator_kit/runner/runner.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022 CERN.
+# Copyright (C) 2022-2025 CERN.
 #
-# Invenio-RDM-Migrator is free software; you can redistribute it and/or modify
+# cds-migrator-kit is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
 
 """InvenioRDM migration streams runner."""

cds_migrator_kit/videos/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2024 CERN.
+# Copyright (C) 2025 CERN.
 #
 # cds-migrator-kit is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.

cds_migrator_kit/videos/migration_config.py

@@ -0,0 +1,49 @@
+"""CDS-Videos settings for CDS-Videos project."""
+
+import json
+import os
+from datetime import datetime, timedelta
+
+
+def _(x):  # needed to avoid start time failure with lazy strings
+    return x
+
+
+# Since HAProxy and Nginx route all requests no matter the host header
+# provided, the allowed hosts variable is set to localhost. In production it
+# should be set to the correct host and it is strongly recommended to only
+# route correct hosts to the application.
+APP_ALLOWED_HOSTS = ["0.0.0.0", "localhost", "127.0.0.1", "localhost.cern.ch"]
+
+SQLALCHEMY_DATABASE_URI = (
+    "postgresql+psycopg2://cds-videos:cds-videos@localhost/cds-videos"
+)
+
+# SECURITY WARNING: keep the secret key used in production secret!
+# Do not commit it to a source code repository.
+# TODO: Set
+SECRET_KEY = "CHANGE_ME"
+
+# TODO: Set with your own hostname when deploying to production
+SITE_UI_URL = "https://127.0.0.1"
+
+SITE_API_URL = "https://127.0.0.1/api"
+
+
+DATACITE_ENABLED = True
+DATACITE_USERNAME = ""
+DATACITE_PASSWORD = ""
+DATACITE_PREFIX = "10.17181"
+DATACITE_TEST_MODE = True
+DATACITE_DATACENTER_SYMBOL = ""
+
+import cds_migrator_kit
+
+base_path = os.path.dirname(os.path.realpath(cds_migrator_kit.__file__))
+logs_dir = os.path.join(base_path, "tmp/logs/")
+CDS_MIGRATOR_KIT_LOGS_PATH = logs_dir
+CDS_MIGRATOR_KIT_VIDEOS_STREAM_CONFIG = (
+    "cds_migrator_kit/videos/weblecture_migration/streams.yaml"
+)
+
+### CDS MIGRATOR #################################

cds_migrator_kit/videos/weblecture_migration/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2024 CERN.
+# Copyright (C) 2025 CERN.
 #
 # CDS-Videos is free software; you can redistribute it and/or modify it under
 # the terms of the MIT License; see LICENSE file for more details.