More type hinting

Author: chezou

.github/workflows/pythontest.yml

@@ -17,7 +17,7 @@ jobs:
         with:
           python-version: "3.13"
       - name: Install dependencies
-        run: uv sync --extra dev
+        run: uv sync --dev
       - name: lint with ruff
         run: |
           uv run ruff format tdclient --diff --exit-non-zero-on-fix

pyproject.toml

@@ -34,11 +34,6 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-dev = [
-  "ruff",
-  "pyright",
-  "tox>=4",
-]
 docs = [
   "sphinx",
   "sphinx_rtd_theme",
@@ -78,15 +73,23 @@ known-third-party = ["dateutil","msgpack","pkg_resources","pytest","setuptools",
 [tool.pyright]
 include = ["tdclient"]
 exclude = ["**/__pycache__", "tdclient/test", "docs"]
-typeCheckingMode = "basic"
+typeCheckingMode = "strict"
 pythonVersion = "3.10"
 pythonPlatform = "All"
-reportMissingTypeStubs = false
-reportUnknownMemberType = false
-reportUnknownArgumentType = false
-reportUnknownVariableType = false
-reportMissingImports = "warning"
+reportMissingTypeStubs = "warning"
+reportUnknownMemberType = "error"
+reportUnknownArgumentType = "error"
+reportMissingImports = "error"
 
 # Pre-commit venv configuration
 venvPath = "."
 venv = ".venv"
+
+[dependency-groups]
+dev = [
+    "ruff",
+    "pyright",
+    "tox>=4",
+    "msgpack-types>=0.5.0",
+    "types-certifi",
+]

tdclient/api.py

@@ -14,11 +14,11 @@
 import time
 import urllib.parse as urlparse
 from array import array
+from collections.abc import Iterator
 from typing import IO, Any, cast
 
 import msgpack
 import urllib3
-import urllib3.util
 
 from tdclient import errors, version
 from tdclient.bulk_import_api import BulkImportAPI
@@ -31,7 +31,7 @@
 from tdclient.schedule_api import ScheduleAPI
 from tdclient.server_status_api import ServerStatusAPI
 from tdclient.table_api import TableAPI
-from tdclient.types import BytesOrStream, StreamBody
+from tdclient.types import BytesOrStream, DataFormat, FileLike, StreamBody
 from tdclient.user_api import UserAPI
 from tdclient.util import (
     csv_dict_record_reader,
@@ -42,7 +42,7 @@
 )
 
 try:
-    import certifi
+    import certifi  # type: ignore[reportMissingImports]
 except ImportError:
     certifi = None
 
@@ -576,7 +576,9 @@ def close(self) -> None:
         # all connections in pool will be closed eventually during gc.
         self.http.clear()
 
-    def _prepare_file(self, file_like, fmt, **kwargs):
+    def _prepare_file(
+        self, file_like: FileLike, fmt: DataFormat, **kwargs: Any
+    ) -> IO[bytes]:
         fp = tempfile.TemporaryFile()
         with contextlib.closing(gzip.GzipFile(mode="wb", fileobj=fp)) as gz:
             packer = msgpack.Packer()
@@ -591,34 +593,41 @@ def _prepare_file(self, file_like, fmt, **kwargs):
         fp.seek(0)
         return fp
 
-    def _read_file(self, file_like, fmt, **kwargs):
+    def _read_file(self, file_like: FileLike, fmt: DataFormat, **kwargs: Any) -> Any:
         compressed = fmt.endswith(".gz")
+        fmt_str = str(fmt)
         if compressed:
-            fmt = fmt[0 : len(fmt) - len(".gz")]
-        reader_name = f"_read_{fmt}_file"
+            fmt_str = fmt_str[0 : len(fmt_str) - len(".gz")]
+        reader_name = f"_read_{fmt_str}_file"
         if hasattr(self, reader_name):
             reader = getattr(self, reader_name)
         else:
             raise TypeError(f"unknown format: {fmt}")
         if hasattr(file_like, "read"):
             if compressed:
-                file_like = gzip.GzipFile(fileobj=file_like)
+                file_like = gzip.GzipFile(fileobj=file_like)  # type: ignore[arg-type]
             return reader(file_like, **kwargs)
         else:
+            # At this point, file_like must be str or bytes (not IO[bytes])
+            file_path = cast("str | bytes", file_like)
             if compressed:
-                file_like = gzip.GzipFile(fileobj=open(file_like, "rb"))
+                file_like = gzip.GzipFile(fileobj=open(file_path, "rb"))  # type: ignore[arg-type]
             else:
-                file_like = open(file_like, "rb")
+                file_like = open(file_path, "rb")
             return reader(file_like, **kwargs)
 
-    def _read_msgpack_file(self, file_like, **kwargs):
+    def _read_msgpack_file(
+        self, file_like: IO[bytes], **kwargs: Any
+    ) -> Iterator[dict[str, Any]]:
         # current impl doesn't tolerate any unpack error
-        unpacker = msgpack.Unpacker(file_like, raw=False)
+        unpacker = msgpack.Unpacker(file_like, raw=False)  # type: ignore[arg-type]
         for record in unpacker:
             validate_record(record)
             yield record
 
-    def _read_json_file(self, file_like, **kwargs):
+    def _read_json_file(
+        self, file_like: IO[bytes], **kwargs: Any
+    ) -> Iterator[dict[str, Any]]:
         # current impl doesn't tolerate any JSON parse error
         for s in file_like:
             record = json.loads(s.decode("utf-8"))
@@ -627,20 +636,22 @@ def _read_json_file(self, file_like, **kwargs):
 
     def _read_csv_file(
         self,
-        file_like,
-        dialect=csv.excel,
-        columns=None,
-        encoding="utf-8",
-        dtypes=None,
-        converters=None,
-        **kwargs,
-    ):
+        file_like: IO[bytes],
+        dialect: type[csv.Dialect] = csv.excel,
+        columns: list[str] | None = None,
+        encoding: str = "utf-8",
+        dtypes: dict[str, Any] | None = None,
+        converters: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ) -> Iterator[dict[str, Any]]:
         if columns is None:
-            reader = csv_dict_record_reader(file_like, encoding, dialect)
+            reader = csv_dict_record_reader(file_like, encoding, dialect)  # type: ignore[arg-type]
         else:
-            reader = csv_text_record_reader(file_like, encoding, dialect, columns)
+            reader = csv_text_record_reader(file_like, encoding, dialect, columns)  # type: ignore[arg-type]
 
         return read_csv_records(reader, dtypes, converters, **kwargs)
 
-    def _read_tsv_file(self, file_like, **kwargs):
+    def _read_tsv_file(
+        self, file_like: IO[bytes], **kwargs: Any
+    ) -> Iterator[dict[str, Any]]:
         return self._read_csv_file(file_like, dialect=csv.excel_tab, **kwargs)

tdclient/bulk_import_api.py

@@ -26,7 +26,7 @@ class BulkImportAPI:
     def get(
         self,
         path: str,
-        params: dict[str, Any] | bytes | None = None,
+        params: dict[str, Any] | None = None,
         headers: dict[str, str] | None = None,
         **kwargs: Any,
     ) -> AbstractContextManager[urllib3.BaseHTTPResponse]: ...
@@ -50,7 +50,7 @@ def raise_error(
     ) -> None: ...
     def checked_json(self, body: bytes, required: list[str]) -> dict[str, Any]: ...
     def _prepare_file(
-        self, file_like: FileLike, fmt: str, **kwargs: Any
+        self, file_like: FileLike, fmt: DataFormat, **kwargs: Any
     ) -> IO[bytes]: ...
 
     def create_bulk_import(
@@ -165,7 +165,7 @@ def validate_part_name(part_name: str) -> None:
             part_name (str): The part name the user is trying to use
         """
         # Check for duplicate periods
-        d = collections.defaultdict(int)
+        d: collections.defaultdict[str, int] = collections.defaultdict(int)
         for char in part_name:
             d[char] += 1
 
@@ -378,5 +378,5 @@ def bulk_import_error_records(
             body = io.BytesIO(res.read())
             decompressor = gzip.GzipFile(fileobj=body)
 
-            unpacker = msgpack.Unpacker(decompressor, raw=False)
+            unpacker = msgpack.Unpacker(decompressor, raw=False)  # type: ignore[arg-type]
             yield from unpacker

tdclient/bulk_import_model.py

@@ -5,7 +5,7 @@
 from typing import TYPE_CHECKING, Any
 
 from tdclient.model import Model
-from tdclient.types import FileLike
+from tdclient.types import BytesOrStream, DataFormat, FileLike
 
 if TYPE_CHECKING:
     from tdclient.client import Client
@@ -112,7 +112,7 @@ def perform(
         self,
         wait: bool = False,
         wait_interval: int = 5,
-        wait_callback: Callable[[], None] | None = None,
+        wait_callback: Callable[["Job"], None] | None = None,
         timeout: float | None = None,
     ) -> "Job":
         """Perform bulk import
@@ -162,7 +162,9 @@ def error_record_items(self) -> Iterator[dict[str, Any]]:
         """
         yield from self._client.bulk_import_error_records(self.name)
 
-    def upload_part(self, part_name: str, bytes_or_stream: FileLike, size: int) -> bool:
+    def upload_part(
+        self, part_name: str, bytes_or_stream: BytesOrStream, size: int
+    ) -> None:
         """Upload a part to bulk import session
 
         Args:
@@ -177,8 +179,8 @@ def upload_part(self, part_name: str, bytes_or_stream: FileLike, size: int) -> b
         return response
 
     def upload_file(
-        self, part_name: str, fmt: str, file_like: FileLike, **kwargs: Any
-    ) -> float:
+        self, part_name: str, fmt: DataFormat, file_like: FileLike, **kwargs: Any
+    ) -> None:
         """Upload a part to Bulk Import session, from an existing file on filesystem.
 
         Args:

tdclient/client.py

@@ -790,12 +790,12 @@ def history(
         """
         result = self.api.history(name, _from or 0, to)
 
-        def scheduled_job(m):
+        def scheduled_job(m: tuple[Any, ...]) -> models.ScheduledJob:
             (
                 scheduled_at,
                 job_id,
                 type,
-                status,
+                _status,
                 query,
                 start_at,
                 end_at,
@@ -837,8 +837,11 @@ def run_schedule(self, name: str, time: int, num: int) -> list[models.ScheduledJ
         """
         results = self.api.run_schedule(name, time, num)
 
-        def scheduled_job(m):
+        def scheduled_job(
+            m: tuple[Any, str, datetime.datetime | None],
+        ) -> models.ScheduledJob:
             job_id, type, scheduled_at = m
+            assert scheduled_at is not None
             return models.ScheduledJob(self, scheduled_at, job_id, type, None)
 
         return [scheduled_job(m) for m in results]
@@ -904,7 +907,7 @@ def results(self) -> list[models.Result]:
         """
         results = self.api.list_result()
 
-        def result(m):
+        def result(m: tuple[str, str, None]) -> models.Result:
             name, url, organizations = m
             return models.Result(self, name, url, organizations)
 
@@ -943,7 +946,7 @@ def users(self):
         """
         results = self.api.list_users()
 
-        def user(m):
+        def user(m: tuple[str, None, None, str]) -> models.User:
             name, org, roles, email = m
             return models.User(self, name, org, roles, email)
 
@@ -1011,7 +1014,7 @@ def close(self) -> None:
 
 
 def job_from_dict(client: Client, dd: dict[str, Any], **values: Any) -> models.Job:
-    d = dict()
+    d: dict[str, Any] = dict()
     d.update(dd)
     d.update(values)
     return models.Job(

tdclient/connection.py

@@ -23,7 +23,7 @@ def __init__(
         wait_callback: Callable[["Cursor"], None] | None = None,
         **kwargs: Any,
     ) -> None:
-        cursor_kwargs = dict()
+        cursor_kwargs: dict[str, Any] = dict()
         if type is not None:
             cursor_kwargs["type"] = type
         if db is not None: