Skip to content

Commit 88c62d3

Browse files
zubeydecivelekzubeydecivelek
committed
setup: separate installation for rdm and videos
1 parent 4170b15 commit 88c62d3

File tree

8 files changed

+280
-34
lines changed

8 files changed

+280
-34
lines changed

‎.github/workflows/tests.yml‎

Lines changed: 51 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,34 +14,34 @@ on:
1414
branches: master
1515
schedule:
1616
# * is a special character in YAML so you have to quote this string
17-
- cron: '0 3 * * 6'
17+
- cron: "0 3 * * 6"
1818
workflow_dispatch:
1919
inputs:
2020
reason:
21-
description: 'Reason'
21+
description: "Reason"
2222
required: false
23-
default: 'Manual trigger'
23+
default: "Manual trigger"
2424

2525
jobs:
26-
Tests:
26+
RDMTests:
2727
runs-on: ubuntu-20.04
2828
strategy:
2929
matrix:
30-
python-version: [3.9]
31-
requirements-level: [pypi]
32-
db-service: [postgresql14]
33-
include:
30+
python-version: [3.9]
31+
requirements-level: [pypi]
32+
db-service: [postgresql14]
33+
include:
3434
- db-service: postgresql14
3535
DB_EXTRAS: "postgresql"
3636

3737
env:
3838
DB: ${{ matrix.db-service }}
39-
EXTRAS: tests
39+
EXTRAS: rdm,tests
4040
steps:
4141
- name: Install python-ldap dependencies
4242
run: |
43-
sudo apt-get update
44-
sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev
43+
sudo apt-get update
44+
sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev
4545
4646
- name: Checkout
4747
uses: actions/checkout@v4
@@ -61,4 +61,43 @@ jobs:
6161
docker compose --version
6262
6363
- name: Run tests
64-
run: ./run-tests.sh
64+
run: ./run-tests.sh tests --ignore=tests/cds-videos
65+
VideosTests:
66+
runs-on: ubuntu-20.04
67+
strategy:
68+
matrix:
69+
python-version: [3.9]
70+
requirements-level: [pypi]
71+
db-service: [postgresql14]
72+
include:
73+
- db-service: postgresql14
74+
DB_EXTRAS: "postgresql"
75+
76+
env:
77+
DB: ${{ matrix.db-service }}
78+
EXTRAS: videos,tests
79+
steps:
80+
- name: Install python-ldap dependencies
81+
run: |
82+
sudo apt-get update
83+
sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev
84+
85+
- name: Checkout
86+
uses: actions/checkout@v4
87+
88+
- name: Set up Python ${{ matrix.python-version }}
89+
uses: actions/setup-python@v5
90+
with:
91+
python-version: ${{ matrix.python-version }}
92+
cache: pip
93+
cache-dependency-path: setup.cfg
94+
95+
- name: Install dependencies
96+
run: |
97+
pip install ".[$EXTRAS]"
98+
pip freeze
99+
docker --version
100+
docker compose --version
101+
102+
- name: Run tests
103+
run: ./run-tests.sh tests/cds-videos

‎README.rst‎

Lines changed: 68 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,80 @@
77
cds-migrator-kit
88
==================
99

10+
Installation
11+
============
1012

11-
TODO change here:
13+
Default Installation (without RDM or Videos)
14+
---------------------------------------------
15+
To install the package without RDM or videos, run:
1216

17+
.. code-block:: bash
1318
14-
Default Installation (without RDM or Videos)
15-
pip install .
19+
pip install .
20+
21+
Installation for RDM
22+
----------------------
23+
To install the package with RDM, run:
24+
25+
.. code-block:: bash
26+
27+
pip install ".[rdm]"
28+
29+
To see available RDM commands, run:
30+
31+
.. code-block:: bash
32+
33+
invenio migration --help
34+
35+
Installation for Videos
36+
-----------------------
37+
To install the package with cds-videos, run:
38+
39+
.. code-block:: bash
40+
41+
pip install ".[videos]"
42+
43+
To see available videos commands, run:
44+
45+
.. code-block:: bash
1646
17-
Install for RDM
47+
invenio migration videos --help
1848
19-
pip install .[rdm]
49+
Running Tests Locally
50+
=====================
2051

21-
Install for Videos
52+
For RDM
53+
--------
54+
Install rdm and test dependencies:
55+
56+
.. code-block:: bash
57+
58+
pip install ".[rdm,tests]"
59+
60+
61+
Run the tests with ignoring `cds-videos` tests:
62+
63+
.. code-block:: bash
64+
65+
./run-tests.sh tests --ignore=tests/cds-videos
66+
67+
For Videos
68+
----------
69+
Install videos and test dependencies:
70+
71+
.. code-block:: bash
72+
73+
pip install ".[videos,tests]"
74+
75+
Run the video tests:
76+
77+
.. code-block:: bash
78+
79+
./run-tests.sh tests/cds-videos
2280
23-
pip install .[videos]
2481
2582
To run the interface:
26-
```
27-
gunicorn -b :8080 --timeout 120 --graceful-timeout 60 cds_migrator_kit.app:app
28-
```
83+
=====================
84+
.. code-block:: bash
85+
gunicorn -b :8080 --timeout 120 --graceful-timeout 60 cds_migrator_kit.app:app
86+

‎cds_migrator_kit/base_config.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# Copyright (C) 2025 CERN.
4+
#
5+
# cds-migrator-kit is free software; you can redistribute it and/or modify it under
6+
# the terms of the MIT License; see LICENSE file for more details.
7+
"""Migration configuration for CDS Migrator Kit."""
8+
9+
import importlib
10+
11+
12+
def import_module(module_name):
13+
"""Try to import a module, return True if successful, otherwise False."""
14+
try:
15+
importlib.import_module(module_name)
16+
return True
17+
except ImportError:
18+
return False
19+
20+
21+
selected_config = None
22+
23+
# Check for `rdm` dependencies
24+
if import_module("cds_rdm.__init__"):
25+
from cds_migrator_kit.rdm import migration_config as selected_config
26+
27+
# Check for `videos` dependencies
28+
elif import_module("cds.version"):
29+
from cds_migrator_kit.videos import migration_config as selected_config
30+
31+
# If no valid module is found, use default one
32+
if selected_config is None:
33+
from cds_migrator_kit import config as selected_config
34+
35+
# Set the selected config module
36+
globals().update(vars(selected_config))

‎cds_migrator_kit/base_minter.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# Copyright (C) 2025 CERN.
4+
#
5+
# cds-migrator-kit is free software; you can redistribute it and/or modify it under
6+
# the terms of the MIT License; see LICENSE file for more details.
7+
"""Minter configuration for CDS Migrator Kit."""
8+
9+
import importlib
10+
import warnings
11+
12+
# Default: No minter
13+
selected_minter = None
14+
15+
# Check if `rdm` is installed and set the minter
16+
try:
17+
importlib.import_module("cds_rdm.__init__")
18+
from cds_rdm.minters import legacy_recid_minter as selected_minter
19+
except ImportError:
20+
warnings.warn(
21+
"No valid PID minter found. Ensure `rdm` is installed.", RuntimeWarning
22+
)
23+
24+
# Expose the minter function
25+
legacy = selected_minter

‎cds_migrator_kit/cli.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# Copyright (C) 2025 CERN.
4+
#
5+
# cds-migrator-kit is free software; you can redistribute it and/or modify it under
6+
# the terms of the MIT License; see LICENSE file for more details.
7+
"""cds-migrator-kit command line module."""
8+
9+
import importlib
10+
11+
import click
12+
13+
14+
def import_module(module_name):
15+
"""Try to import a module, return True if successful, otherwise False."""
16+
try:
17+
importlib.import_module(module_name)
18+
return True
19+
except ImportError:
20+
return False
21+
22+
23+
@click.group()
24+
def cli():
25+
"""Base CLI command that loads the subcommands."""
26+
pass
27+
28+
29+
# Check for `rdm` dependencies
30+
if import_module("cds_rdm.__init__"):
31+
from cds_migrator_kit.rdm.cli import migration
32+
33+
cli = migration
34+
35+
# Check for `videos` dependencies
36+
if import_module("cds.version"):
37+
from cds_migrator_kit.videos.weblecture_migration.cli import videos
38+
39+
cli.add_command(videos, "videos")

‎cds_migrator_kit/rdm/migration_config.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -368,9 +368,6 @@ def _(x): # needed to avoid start time failure with lazy strings
368368
CDS_MIGRATOR_KIT_LOGS_PATH = logs_dir
369369

370370
CDS_MIGRATOR_KIT_STREAM_CONFIG = "cds_migrator_kit/rdm/streams.yaml"
371-
CDS_MIGRATOR_KIT_VIDEOS_STREAM_CONFIG = (
372-
"cds_migrator_kit/videos/weblecture_migration/streams.yaml"
373-
)
374371

375372
RDM_RECORDS_IDENTIFIERS_SCHEMES = {
376373
**RDM_RECORDS_IDENTIFIERS_SCHEMES,

‎cds_migrator_kit/videos/migration_config.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
"""CDS-Videos settings for CDS-Videos project."""
2+
3+
import json
4+
import os
5+
from datetime import datetime, timedelta
6+
7+
8+
def _(x): # needed to avoid start time failure with lazy strings
9+
return x
10+
11+
12+
# Since HAProxy and Nginx route all requests no matter the host header
13+
# provided, the allowed hosts variable is set to localhost. In production it
14+
# should be set to the correct host and it is strongly recommended to only
15+
# route correct hosts to the application.
16+
APP_ALLOWED_HOSTS = ["0.0.0.0", "localhost", "127.0.0.1", "localhost.cern.ch"]
17+
18+
SQLALCHEMY_DATABASE_URI = (
19+
"postgresql+psycopg2://cds-videos:cds-videos@localhost/cds-videos"
20+
)
21+
22+
# SECURITY WARNING: keep the secret key used in production secret!
23+
# Do not commit it to a source code repository.
24+
# TODO: Set
25+
SECRET_KEY = "CHANGE_ME"
26+
27+
# TODO: Set with your own hostname when deploying to production
28+
SITE_UI_URL = "https://127.0.0.1"
29+
30+
SITE_API_URL = "https://127.0.0.1/api"
31+
32+
33+
DATACITE_ENABLED = True
34+
DATACITE_USERNAME = ""
35+
DATACITE_PASSWORD = ""
36+
DATACITE_PREFIX = "10.17181"
37+
DATACITE_TEST_MODE = True
38+
DATACITE_DATACENTER_SYMBOL = ""
39+
40+
import cds_migrator_kit
41+
42+
base_path = os.path.dirname(os.path.realpath(cds_migrator_kit.__file__))
43+
logs_dir = os.path.join(base_path, "tmp/logs/")
44+
CDS_MIGRATOR_KIT_LOGS_PATH = logs_dir
45+
CDS_MIGRATOR_KIT_VIDEOS_STREAM_CONFIG = (
46+
"cds_migrator_kit/videos/weblecture_migration/streams.yaml"
47+
)
48+
49+
### CDS MIGRATOR #################################

0 commit comments

Comments
 (0)