Skip to content

[DOP-22129] Allow using etl-entities 2.5.0 #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dolfinus merged 1 commit into from
Jan 28, 2025
Merged

[DOP-22129] Allow using etl-entities 2.5.0 #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
include:
- python-version: '3.7'
pydantic-version: '1'
os: ubuntu-latest
os: ubuntu-22.04

- python-version: '3.13'
pydantic-version: '2'
Expand Down
193 changes: 115 additions & 78 deletions conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import os
import secrets
from collections import namedtuple
from datetime import date, datetime
from datetime import date, datetime, timezone
from pathlib import Path
from unittest.mock import Mock

import etl_entities
import pytest
from etl_entities.hwm import (
ColumnDateHWM,
Expand All @@ -13,6 +16,7 @@
)
from horizon.client.auth import LoginPassword
from horizon.commons.schemas.v1 import NamespaceCreateRequestV1
from packaging.version import Version

from horizon_hwm_store import HorizonHWMStore

Expand All @@ -25,93 +29,126 @@
HORIZON_PASSWORD = os.environ.get("HORIZON_PASSWORD")
HORIZON_NAMESPACE = os.environ.get("HORIZON_NAMESPACE")


@pytest.fixture(
params=[
(
ColumnIntHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
# no source
expression=secrets.token_hex(5),
value=10,
),
15,
HWMS_WITH_VALUE = [
(
ColumnIntHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}", # noqa: WPS204
# no source
expression=secrets.token_hex(5),
value=10,
),
(
ColumnIntHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
source=secrets.token_hex(5),
expression=secrets.token_hex(5),
value=10,
),
15,
15,
),
(
ColumnIntHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
source=secrets.token_hex(5),
expression=secrets.token_hex(5),
value=10,
),
(
ColumnDateHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
source=secrets.token_hex(5),
expression=secrets.token_hex(5),
value=date(year=2023, month=8, day=15),
),
date(year=2023, month=9, day=15), # + 1 month
15,
),
(
ColumnDateHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
source=secrets.token_hex(5),
expression=secrets.token_hex(5),
value=date(year=2023, month=8, day=15),
),
(
ColumnDateTimeHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
source=secrets.token_hex(5),
expression=secrets.token_hex(5),
value=datetime(year=2023, month=8, day=15, hour=11, minute=22, second=33),
),
datetime(year=2023, month=8, day=15, hour=11, minute=23, second=33), # + 1 minute
date(year=2023, month=9, day=15), # + 1 month
),
(
ColumnDateTimeHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
source=secrets.token_hex(5),
expression=secrets.token_hex(5),
value=datetime(year=2023, month=8, day=15, hour=11, minute=22, second=33),
),
(
FileListHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
# no directory
value=["/some/file1", "/another/file2"],
),
["/some/file1", "/another/file2", "/more/file3"],
datetime(year=2023, month=8, day=15, hour=11, minute=23, second=33), # + 1 minute
),
(
FileListHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
# no directory
value=["/some/file1", "/another/file2"],
),
(
FileListHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
directory="/absolute/path",
value=["/absolute/path/file1", "/absolute/path/file2"],
),
["/absolute/path/file1", "/absolute/path/file2", "/absolute/path/file3"],
["/some/file1", "/another/file2", "/more/file3"],
),
(
FileListHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
directory="/absolute/path",
value=["/absolute/path/file1", "/absolute/path/file2"],
),
(
KeyValueIntHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
# no topic
expression="offset",
value={
0: 100,
1: 123,
},
),
{
0: 110,
1: 150,
["/absolute/path/file1", "/absolute/path/file2", "/absolute/path/file3"],
),
(
KeyValueIntHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
# no topic
expression="offset",
value={
0: 100,
1: 123,
},
),
(
KeyValueIntHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
topic="topic_name",
expression="offset",
value={
0: 100,
1: 123,
},
),
{
0: 110,
1: 150,
{
0: 110,
1: 150,
},
),
(
KeyValueIntHWM(
name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
topic="topic_name",
expression="offset",
value={
0: 100,
1: 123,
},
),
],
)
{
0: 110,
1: 150,
},
),
]

if Version(etl_entities.__version__) >= Version("2.5.0"):
from etl_entities.hwm import FileModifiedTimeHWM

def file_with_mtime(mtime: datetime) -> Path:
result = Mock(spec=Path)
result.exists.return_value = True
result.is_file.return_value = True
result_stat = Mock(spec=os.stat_result)
result_stat.st_mtime = mtime.timestamp()
result.stat.return_value = result_stat
return result

HWMS_WITH_VALUE.extend(
[
(
FileModifiedTimeHWM(
name=secrets.token_hex(5),
# no directory
value=datetime(2025, 1, 1, 11, 22, 33, 456789, tzinfo=timezone.utc),
),
file_with_mtime(datetime(2025, 1, 1, 22, 33, 44, 567890, tzinfo=timezone.utc)),
),
(
FileModifiedTimeHWM(
name=secrets.token_hex(5),
directory="/absolute/path",
value=datetime(2025, 1, 1, 11, 22, 33, 456789, tzinfo=timezone.utc),
),
file_with_mtime(datetime(2025, 1, 1, 22, 33, 44, 567890, tzinfo=timezone.utc)),
),
],
)


@pytest.fixture(params=HWMS_WITH_VALUE)
def hwm_new_value(request):
return request.param

Expand Down
1 change: 1 addition & 0 deletions docs/changelog/next_release/40.feature.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Allow using `etl-entities==2.5.0 <https://github.com/MobileTeleSystems/etl-entities/releases/tag/2.5.0>`_.
2 changes: 1 addition & 1 deletion requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
data-horizon[client-sync]>=1.0.0,<1.1
etl-entities>=2.1.0,<2.5.0
etl-entities>=2.1.0,<2.6.0
Loading