Merge branch 'develop'

Ilyas Gasanov · Ilyas Gasanov · commit 61fd4f7dca34 · 2025-02-26T18:36:50.000+03:00
diff --git a/.env.docker b/.env.docker
@@ -1,18 +1,6 @@
-# Postgres
-POSTGRES_DB=horizon
-POSTGRES_USER=horizon
-POSTGRES_PASSWORD=I8p4E2mULz6v
-
-# Horizon
-HORIZON__DATABASE__URL=postgresql+asyncpg://horizon:I8p4E2mULz6v@db:5432/horizon
-HORIZON__AUTH__ACCESS_TOKEN__SECRET_KEY=bae1thahr8Iyaisai0kohvoh1aeg5quu
-HORIZON__AUTH__PROVIDER=horizon.backend.providers.auth.dummy.DummyAuthProvider
-HORIZON__SERVER__LOGGING__PRESET=colored
-HORIZON__SERVER__DEBUG=false
-
 # envs for Horizon integration tests
 HORIZON_HOST=horizon
 HORIZON_PORT=8000
 HORIZON_USER=horizon
-HORIZON_PASSWORD=password
+HORIZON_PASSWORD=345UsedForTestOnly
 HORIZON_NAMESPACE=namespace
diff --git a/.env.local b/.env.local
@@ -1,6 +1,6 @@
-# Horizon
+# envs for Horizon integration tests
 export HORIZON_HOST=localhost
 export HORIZON_PORT=8000
 export HORIZON_USER=horizon
-export HORIZON_PASSWORD=password
+export HORIZON_PASSWORD=345UsedForTestOnly
 export HORIZON_NAMESPACE=namespace
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -111,4 +111,5 @@ jobs:
           name: ${{ steps.release-name.outputs.name }}
           body_path: changelog.md
           files: |
-            dist/*
+            dist/*.tar.gz
+            dist/*.whl
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -43,7 +43,7 @@ repos:
           - --no-extra-eol
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.0
+    rev: v2.4.1
     hooks:
       - id: codespell
         args: [-w]
@@ -88,7 +88,7 @@ repos:
       - id: add-trailing-comma
 
   - repo: https://github.com/pycqa/bandit
-    rev: 1.8.2
+    rev: 1.8.3
     hooks:
       - id: bandit
         args:
@@ -98,7 +98,7 @@ repos:
         require_serial: true
 
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.10.0
+    rev: 25.1.0
     hooks:
       - id: black
 
@@ -120,7 +120,7 @@ repos:
       - id: isort
 
   - repo: https://github.com/pycqa/bandit
-    rev: 1.8.2
+    rev: 1.8.3
     hooks:
       - id: bandit
         args:
diff --git a/conftest.py b/conftest.py
@@ -15,7 +15,6 @@
     KeyValueIntHWM,
 )
 from horizon.client.auth import LoginPassword
-from horizon.commons.schemas.v1 import NamespaceCreateRequestV1
 from packaging.version import Version
 
 from horizon_hwm_store import HorizonHWMStore
@@ -149,6 +148,11 @@ def file_with_mtime(mtime: datetime) -> Path:
 
 
 @pytest.fixture(params=HWMS_WITH_VALUE)
+def hwm_new_values(request):
+    return request.param
+
+
+@pytest.fixture(params=[HWMS_WITH_VALUE[0]])
 def hwm_new_value(request):
     return request.param
 
@@ -164,16 +168,9 @@ def hwm_store():
 
 @pytest.fixture(scope="module")
 def ensure_namespace():
-    from requests.exceptions import HTTPError
 
-    store = HorizonHWMStore(
+    HorizonHWMStore(
         api_url=HORIZON_URL,
         auth=LoginPassword(login=HORIZON_USER, password=HORIZON_PASSWORD),
         namespace=HORIZON_NAMESPACE,
-    )
-
-    try:
-        store.client.create_namespace(NamespaceCreateRequestV1(name=HORIZON_NAMESPACE))  # noqa: WPS437
-    except HTTPError:
-        # exception: 409 Client Error: Conflict for url: http://horizon/v1/namespaces/ - namespace already exists
-        pass
+    ).force_create_namespace()
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -3,7 +3,10 @@ services:
   db:
     image: postgres:15-alpine
     restart: unless-stopped
-    env_file: .env.docker
+    environment:
+      POSTGRES_DB: horizon
+      POSTGRES_USER: horizon
+      POSTGRES_PASSWORD: 123UsedForTestOnly
     ports:
       - 5432:5432
     volumes:
@@ -18,7 +21,12 @@ services:
   backend:
     image: mtsrus/horizon-backend:latest
     restart: unless-stopped
-    env_file: .env.docker
+    environment:
+      HORIZON__DATABASE__URL: postgresql+asyncpg://horizon:123UsedForTestOnly@db:5432/horizon
+      HORIZON__AUTH__ACCESS_TOKEN__SECRET_KEY: 234UsedForTestOnly
+      HORIZON__AUTH__PROVIDER: horizon.backend.providers.auth.dummy.DummyAuthProvider
+      HORIZON__SERVER__LOGGING__PRESET: colored
+      HORIZON__SERVER__DEBUG: 'false'
     ports:
       - 8000:8000
     depends_on:
diff --git a/docs/changelog/1.1.0.rst b/docs/changelog/1.1.0.rst
@@ -0,0 +1,10 @@
+
+
+
+1.1.0 (2025-02-26)
+==================
+
+Features
+--------
+
+- Add ``force_create_namespace`` method (:github:pull:`44`)
diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst
@@ -3,6 +3,7 @@
     :caption: Changelog
 
     DRAFT
+    1.1.0
     1.0.2
     1.0.1
     1.0.0
diff --git a/docs/horizon-hwm-store.rst b/docs/horizon-hwm-store.rst
@@ -6,4 +6,4 @@ Horizon HWM Store
 .. currentmodule:: horizon_hwm_store.horizon_hwm_store
 
 .. autoclass:: HorizonHWMStore
-    :members: get_hwm, set_hwm
+    :members: get_hwm, set_hwm, force_create_namespace, check
diff --git a/horizon_hwm_store/VERSION b/horizon_hwm_store/VERSION
@@ -1 +1 @@
-1.0.2
+1.1.0
diff --git a/horizon_hwm_store/horizon_hwm_store.py b/horizon_hwm_store/horizon_hwm_store.py
@@ -8,11 +8,14 @@
 from etl_entities.hwm_store import BaseHWMStore, register_hwm_store_class
 from horizon.client.auth import LoginPassword
 from horizon.client.sync import HorizonClientSync, RetryConfig, TimeoutConfig
+from horizon.commons.exceptions import EntityAlreadyExistsError
 from horizon.commons.schemas.v1 import (
     HWMCreateRequestV1,
     HWMPaginateQueryV1,
     HWMUpdateRequestV1,
+    NamespaceCreateRequestV1,
     NamespacePaginateQueryV1,
+    NamespaceResponseV1,
 )
 
 try:
@@ -33,7 +36,7 @@ class HorizonHWMStore(BaseHWMStore):
 
     .. warning::
 
-        It is required to create namespace in Horizon BEFORE using this class.
+        It is required to create a namespace BEFORE working with HWMs.
 
     Parameters
     ----------
@@ -98,7 +101,7 @@ class HorizonHWMStore(BaseHWMStore):
             namespace="namespace",
             retry=RetryConfig(total=5),
             timeout=TimeoutConfig(request_timeout=10),
-        ):
+        ).force_create_namespace():
             with IncrementalStrategy():
                 df = reader.run()
                 writer.run(df)
@@ -213,6 +216,24 @@ def check(self) -> HorizonHWMStore:
         self._get_namespace_id()
         return self
 
+    def force_create_namespace(self) -> HorizonHWMStore:
+        """
+        Create a namespace with name specified in HorizonHWMStore class.
+
+        Returns
+        -------
+        HorizonHWMStore
+            Self
+        """
+        namespace = self._get_namespace(self.namespace)
+        if namespace is None:
+            try:
+                namespace = self.client.create_namespace(NamespaceCreateRequestV1(name=self.namespace))
+                self._namespace_id = namespace.id  # noqa: WPS601
+            except EntityAlreadyExistsError:
+                ...
+        return self
+
     # LoginPassword, RetryConfig and TimeoutConfig can be inherited from Pydantic v2 BaseModel
     # which is detected by Pydantic v1 as arbitrary type. So we need to parse them manually.
     @validator("auth", pre=True)
@@ -233,6 +254,10 @@ def _check_timeout(cls, value: TimeoutConfig):
             return TimeoutConfig.parse_obj(value)
         return value
 
+    def _get_namespace(self, name: str) -> NamespaceResponseV1 | None:
+        namespaces = self.client.paginate_namespaces(query=NamespacePaginateQueryV1(name=name)).items
+        return namespaces[0] if namespaces else None
+
     def _get_namespace_id(self) -> int:
         """
         Fetch the ID of the namespace. Raises an exception if the namespace doesn't exist.
@@ -250,11 +275,14 @@ def _get_namespace_id(self) -> int:
         if self._namespace_id is not None:
             return self._namespace_id
 
-        namespaces = self.client.paginate_namespaces(NamespacePaginateQueryV1(name=self.namespace)).items
-        if not namespaces:
-            raise RuntimeError(f"Namespace {self.namespace!r} not found. Please create it before using.")
+        namespace = self._get_namespace(self.namespace)
+        if namespace is None:
+            raise RuntimeError(
+                f"Namespace {self.namespace!r} not found. "
+                "Please create it before using by calling .force_create_namespace() method.",
+            )
 
-        self._namespace_id = namespaces[0].id  # noqa: WPS601
+        self._namespace_id = namespace.id  # noqa: WPS601
         return self._namespace_id
 
     def _get_hwm_id(self, namespace_id: int, hwm_name: str) -> Optional[int]:
diff --git a/tests/tests_integration/test_horizon_hwm_store_integration.py b/tests/tests_integration/test_horizon_hwm_store_integration.py
@@ -14,8 +14,8 @@
 HORIZON_NAMESPACE = os.environ.get("HORIZON_NAMESPACE")
 
 
-def test_hwm_store_integration(hwm_store, hwm_new_value, ensure_namespace):
-    hwm, new_value = hwm_new_value
+def test_hwm_store_integration(hwm_store, hwm_new_values, ensure_namespace):
+    hwm, new_value = hwm_new_values
     assert hwm_store.get_hwm(hwm.name) is None
 
     hwm_store.set_hwm(hwm)
@@ -65,3 +65,19 @@ def test_hwm_store_unexisting_namespace(hwm_new_value):
 
     with pytest.raises(RuntimeError, match=error_msg):
         store.set_hwm(hwm)
+
+
+def test_hwm_store_creates_namespace():
+    namespace_name = "new_namespace"
+    store = HorizonHWMStore(
+        api_url=HORIZON_URL,
+        auth=LoginPassword(login=HORIZON_USER, password=HORIZON_PASSWORD),
+        namespace=namespace_name,
+    )
+    error_msg = "Namespace 'new_namespace' not found. Please create it before using."
+
+    with pytest.raises(RuntimeError, match=error_msg):
+        store.get_hwm("some_hwm_name")
+
+    store.force_create_namespace()
+    assert store.get_hwm("some_hwm_name") is None

-Original file line number
+Diff line change
     :caption: Changelog
     DRAFT
 +    1.1.0
 .0.2
 .0.1
 .0.0