[VREE-168] Add endpoints to start ability evaluation directly from Da…

examples/import_dataset.py

@@ -10,8 +10,17 @@
 from eyepop import EyePopSdk
 from eyepop.data.data_endpoint import DataEndpoint
 from eyepop.data.data_jobs import DataJob
-from eyepop.data.data_types import Asset, Dataset, DatasetCreate, AutoAnnotate, DatasetAutoAnnotate, \
-    DatasetAutoAnnotateCreate, AutoAnnotateStatus, Prediction, PredictedClass, UserReview
+from eyepop.data.data_types import (
+    Asset,
+    AutoAnnotate,
+    AutoAnnotateStatus,
+    Dataset,
+    DatasetAutoAnnotateCreate,
+    DatasetCreate,
+    PredictedClass,
+    Prediction,
+    UserReview,
+)
 
 logging.getLogger('eyepop').setLevel(logging.DEBUG)
 logging.getLogger('eyepop.requests').setLevel(logging.DEBUG)
@@ -19,7 +28,7 @@
 
 log = logging.getLogger(__name__)
 
 class LocalAsset(Callable[[], Any]):
     def __init__(self, path: str) -> None:
         self.path = path
         self.file = None
@@ -100,22 +109,6 @@
         )
     return len(assets)
 
-async def get_auto_annotate(
-        endpoint: DataEndpoint,
-        dataset: Dataset,
-        auto_annotate: AutoAnnotate,
-        source: str
-) -> DatasetAutoAnnotate:
-    pass
-
-async def get_auto_annotate(
-        endpoint: DataEndpoint,
-        dataset: Dataset,
-        auto_annotate: AutoAnnotate,
-        source: str
-) -> DatasetAutoAnnotate:
-    pass
-
 parser = argparse.ArgumentParser(
                     prog='import dataset',
                     description='Importing local assets into a dataset',
@@ -163,7 +156,7 @@
             log.debug("approved %d asset annotations in dataset: %s", num_assets, dataset.uuid)
         finally:
             if main_args.dataset_uuid is None and main_args.remove:
                 log.debug("removing dataset: %s", dataset.uuid)
                 await endpoint.delete_dataset(dataset.uuid)
 
 asyncio.run(main())

examples/infer_demo.py

@@ -7,7 +7,12 @@
 from eyepop import EyePopSdk
 from eyepop.data.data_endpoint import DataEndpoint
 from eyepop.data.data_jobs import InferJob
-from eyepop.data.data_types import InferRequest, TranscodeMode, Asset, AssetInclusionMode, EvaluateRequest
+from eyepop.data.data_types import (
+    Asset,
+    EvaluateRequest,
+    InferRequest,
+    TranscodeMode,
+)
 
 script_dir = os.path.dirname(__file__)
 

examples/pop_demo.py

@@ -13,6 +13,7 @@
 from dotenv import load_dotenv
 from PIL import Image
 from pybars import Compiler
+from pydantic import TypeAdapter
 from webui import webui
 
 from eyepop import EyePopSdk, Job
@@ -243,7 +244,7 @@
     ]),
 }
 
 def list_of_points(arg: str) -> list[dict[str, any]]:
     points = []
     points_as_tuples = ast.literal_eval(f'[{arg}]')
     for tuple in points_as_tuples:
@@ -254,7 +255,7 @@
     return points
 
 
 def list_of_boxes(arg: str) -> list[dict[str, any]]:
     boxes = []
     boxes_as_tuples = ast.literal_eval(f'[{arg}]')
     for tuple in boxes_as_tuples:
@@ -307,6 +308,7 @@
 parser.add_argument('-po', '--points', required=False, type=list_of_points, help="List of POIs as coordinates like (x1, y1), (x2, y2) in the original image coordinate system")
 parser.add_argument('-bo', '--boxes', required=False, type=list_of_boxes, help="List of POIs as boxes like (left1, top1, right1, bottom1), (left1, top1, right1, bottom1) in the original image coordinate system")
 parser.add_argument('-sp', '--single-prompt', required=False, type=str, help="Single prompt to pass as parameter")
+parser.add_argument('-sl', '--single-label', required=False, type=str, help="Single label to use as result for single prompt to pass as parameter")
 parser.add_argument('-pr', '--prompt', required=False, type=str, help="Prompt to pass as parameter", action="append")
 parser.add_argument('-v', '--visualize', required=False, help="show rendered output", default=False, action="store_true")
 parser.add_argument('-o', '--output', required=False, help="print results to stdout", default=False, action="store_true")
@@ -359,7 +361,7 @@
             id=i+1,
             ability=alias,
             targetFps = main_args.fps,
-    ) for i, alias in enumerate(main_args.model_alias)
+        ) for i, alias in enumerate(main_args.model_alias)
     ])
     if main_args.top_k is not None:
         for c in pop.components:
@@ -437,7 +439,8 @@
 elif main_args.single_prompt is not None:
     params = [
         ComponentParams(componentId=1, values={
-            "prompt": main_args.single_prompt
+            "prompt": main_args.single_prompt,
+            "label": main_args.single_label if main_args.single_label else main_args.single_prompt
         })
     ]
 
@@ -446,7 +449,10 @@
     visualize_path = None
     example_image_src = None
     if args.dump:
-        print(pop.model_dump_json(indent=2))
+        print("Pop:", pop.model_dump_json(indent=2))
+        if params:
+            print("Params:", TypeAdapter(list[ComponentParams]).dump_json(params, indent=2).decode("utf-8"))
+
     async with EyePopSdk.workerEndpoint(dataset_uuid=args.dataset_uuid, is_async=True) as endpoint:
         await endpoint.set_pop(pop)
         if args.local_path:

eyepop/data/data_endpoint.py

@@ -12,8 +12,10 @@
 
 from eyepop.client_session import ClientSession
 from eyepop.data.arrow.schema import MIME_TYPE_APACHE_ARROW_FILE_VERSIONED
-from eyepop.data.data_jobs import DataJob, InferJob, _ImportFromJob, _UploadStreamJob, EvaluateJob
+from eyepop.data.data_jobs import DataJob, EvaluateJob, InferJob, _ImportFromJob, _UploadStreamJob
 from eyepop.data.data_types import (
+    APPLICATION_JSON,
+    AliasResolution,
     AnnotationInclusionMode,
     ArgoWorkflowPhase,
     ArtifactType,
@@ -33,6 +35,7 @@
     DatasetCreate,
     DatasetUpdate,
     DownloadResponse,
+    EvaluateRequest,
     EventHandler,
     ExportedUrlResponse,
     InferRequest,
@@ -50,8 +53,13 @@
     Prediction,
     QcAiHubExportParams,
     TranscodeMode,
-    UserReview, EvaluateRequest, APPLICATION_JSON, VlmAbilityGroupResponse, VlmAbilityGroupCreate,
-    VlmAbilityGroupUpdate, VlmAbilityResponse, VlmAbilityCreate, VlmAbilityUpdate, AliasResolution,
+    UserReview,
+    VlmAbilityCreate,
+    VlmAbilityGroupCreate,
+    VlmAbilityGroupResponse,
+    VlmAbilityGroupUpdate,
+    VlmAbilityResponse,
+    VlmAbilityUpdate,
 )
 from eyepop.endpoint import Endpoint, log_requests
 from eyepop.settings import settings
@@ -1212,3 +1220,24 @@ async def remove_vlm_ability_alias(
         delete_url = f'{await self.data_base_url()}/vlm_abilities/{vlm_ability_uuid}/alias/{alias_name}/tag/{tag_name}'
         async with await self.request_with_retry("DELETE", delete_url) as resp:
             return parse_obj_as(VlmAbilityResponse, await resp.json()) # type: ignore [no-any-return]
+
+    async def list_vlm_ability_evaluations(self, vlm_ability_uuid: str) -> list[DatasetAutoAnnotate]:
+        get_url = f'{await self.data_base_url()}/vlm_abilities/{vlm_ability_uuid}/evaluations'
+        async with await self.request_with_retry("GET", get_url) as resp:
+            return parse_obj_as(list[DatasetAutoAnnotate], await resp.json()) # type: ignore [no-any-return]
+
+    async def get_vlm_ability_evaluation(self, vlm_ability_uuid: str, source: str) -> DatasetAutoAnnotate:
+        get_url = f'{await self.data_base_url()}/vlm_abilities/{vlm_ability_uuid}/evaluations/{source}'
+        async with await self.request_with_retry("GET", get_url) as resp:
+            return parse_obj_as(DatasetAutoAnnotate, await resp.json()) # type: ignore [no-any-return]
+
+    async def start_vlm_ability_evaluation(self, vlm_ability_uuid: str, evaluate_request: EvaluateRequest) -> DatasetAutoAnnotate:
+        post_url = f'{await self.data_base_url()}/vlm_abilities/{vlm_ability_uuid}/evaluations'
+        async with await self.request_with_retry(
+                method="POST",
+                url=post_url,
+                content_type=APPLICATION_JSON,
+                data=evaluate_request.model_dump_json(exclude_none=True)
+        ) as resp:
+            return parse_obj_as(DatasetAutoAnnotate, await resp.json()) # type: ignore [no-any-return]
+

eyepop/data/data_jobs.py

@@ -8,8 +8,16 @@
 from pydantic import BaseModel, Field
 
 from eyepop.client_session import ClientSession
-from eyepop.data.data_types import Asset, AssetImport, InferRequest, Prediction, EvaluateRequest, EvaluateResponse, \
-    InferRunInfo, APPLICATION_JSON
+from eyepop.data.data_types import (
+    APPLICATION_JSON,
+    Asset,
+    AssetImport,
+    EvaluateRequest,
+    EvaluateResponse,
+    InferRequest,
+    InferRunInfo,
+    Prediction,
+)
 from eyepop.jobs import Job, JobStateCallback
 
 
@@ -269,6 +277,6 @@
                     break
                 else:
                     raise ValueError(f"Unexpected status code: {resp.status}")
         if result is None:
             raise TimeoutError(f"evaluate request timed out after {time.time() - start_time} seconds")
 

eyepop/data/data_syncify.py

@@ -5,8 +5,9 @@
 import aiohttp
 
 from eyepop.data.data_endpoint import DataEndpoint
-from eyepop.data.data_jobs import DataJob, InferJob, EvaluateJob
+from eyepop.data.data_jobs import DataJob, EvaluateJob, InferJob
 from eyepop.data.data_types import (
+    AliasResolution,
     AnnotationInclusionMode,
     ArtifactType,
     Asset,
@@ -23,9 +24,12 @@
     DatasetCreate,
     DatasetUpdate,
     DownloadResponse,
+    EvaluateRequest,
+    EvaluateResponse,
     EventHandler,
     ExportedUrlResponse,
     InferRequest,
+    InferRunInfo,
     Model,
     ModelAlias,
     ModelAliasCreate,
@@ -39,8 +43,13 @@
     Prediction,
     QcAiHubExportParams,
     TranscodeMode,
-    UserReview, EvaluateResponse, InferRunInfo, VlmAbilityGroupResponse, VlmAbilityGroupCreate, VlmAbilityGroupUpdate,
-    VlmAbilityResponse, VlmAbilityCreate, VlmAbilityUpdate, EvaluateRequest, AliasResolution,
+    UserReview,
+    VlmAbilityCreate,
+    VlmAbilityGroupCreate,
+    VlmAbilityGroupResponse,
+    VlmAbilityGroupUpdate,
+    VlmAbilityResponse,
+    VlmAbilityUpdate,
 )
 from eyepop.syncify import SyncEndpoint, run_coro_thread_save
 

eyepop/data/types/asset.py

@@ -14,6 +14,7 @@ class AssetAnnotation(BaseModel):
     approved_threshold: float | None = None
     auto_annotate: AutoAnnotate | None = None
     source: str | None = None
+    source_ability_uuid: str | None = None
     predictions: Sequence[Prediction] | None = None
     uncertainty_score: float | None = None
 

eyepop/data/types/dataset.py

@@ -108,6 +108,7 @@ class DatasetAutoAnnotate(BaseModel):
     updated_at: datetime | None = None
     dataset_uuid: str
     dataset_version: int
+    source_ability_uuid: str | None = None
     source_model_uuid: str | None = None
     auto_annotate: AutoAnnotate | None = None
     auto_annotate_params: dict[str, Any] | None = None

eyepop/data/types/vlm.py

@@ -97,6 +97,10 @@ class EvaluateRequest(BaseModel):
         default=None,
         description="VLM inference config; mutually exclusive to ability_uuid",
     )
+    source: str | None = Field(
+        default=None,
+        description="Optional source identifier to be used for the auto annotations",
+    )
     dataset_uuid: str = Field(description="The Uuid dataset to evaluate.")
     filter: EvaluateFilter | None = Field(
         default=None,

eyepop/endpoint.py

@@ -2,7 +2,7 @@
 import logging
 import time
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Awaitable
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Type
 
 import aiohttp
 
@@ -111,13 +111,16 @@
         self.retry_handlers[status_code] = handler
 
     def __enter__(self) -> None:
+        """Not implemented, use async with instead."""
         raise TypeError("Use async with instead")
 
     def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException],
                  exc_tb: Optional[TracebackType], ) -> None:
+        """Not implemented, use async with instead."""
         pass  # pragma: no cover
 
     async def __aenter__(self) -> "Endpoint":
+        """Connect."""
         try:
             await self.connect()
         except aiohttp.ClientError as e:
@@ -128,6 +131,7 @@
 
     async def __aexit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException],
                         exc_tb: Optional[TracebackType], ) -> None:
+        """Disconnect."""
         await self.disconnect()
 
     async def _authorization_header(self) -> str | None:
@@ -154,7 +158,7 @@
             await asyncio.gather(*tasks)
 
         if self.request_tracer and self.client_session:
             await self.event_sender.stop()
             if self.compute_ctx is None:
                 await self.request_tracer.send_and_reset(f'{self.eyepop_url}/events', await self._authorization_header(),
                                                          None)
@@ -182,7 +186,7 @@
         try:
             await self._reconnect()
         except Exception as e:
             await self.client_session.close()
             self.client_session = None
             raise e
 
@@ -218,7 +222,7 @@
                     'Content-Type': 'application/json',
                     'Accept': 'application/json'
                 }
                 async with self.client_session.post(authenticate_url, headers=api_auth_header) as response:
                     response_json = await response.json()
                     self.compute_ctx.m2m_access_token = response_json['access_token']
                     log.debug(f"compute ctx m2m access token: {self.compute_ctx.m2m_access_token}")
@@ -226,7 +230,7 @@
         if self.secret_key is None:
             return None
         now = time.time()
         if self.token is None or self.expire_token_time < now:
             body = {'secret_key': self.secret_key}
             post_url = f'{self.eyepop_url}/authentication/token'
             log_requests.debug('before POST %s', post_url)

eyepop/metrics.py

@@ -34,7 +34,7 @@ def get_average_times(self) -> dict:
 
     def update_count_by_state(self, state: JobState):
         count = 0
-        for job, job_state in self.jobs_to_state.items():
+        for _, job_state in self.jobs_to_state.items():
             if state == job_state:
                 count += 1
         if count > self.max_number_of_jobs_by_state[state]:
@@ -101,4 +101,5 @@ class JobState(Enum):
     DRAINED = 6
 
     def __repr__(self):
+        """To string."""
         return self._name_

eyepop/syncify.py

@@ -5,22 +5,28 @@
 import types
 import typing
 from asyncio import StreamReader
+from typing import TYPE_CHECKING
 
 log = logging.getLogger(__name__)
 
+if TYPE_CHECKING:
+    from eyepop import Endpoint
 
 class SyncEndpoint:
     def __init__(self, endpoint: "Endpoint"):
+        """Constructor."""
         self._on_ready = None
         self.endpoint = endpoint
         self.event_loop = asyncio.new_event_loop()
         self.thread = threading.Thread(target=self._run_event_loop, args=(self.event_loop,), daemon=True)
         self.thread.start()
 
     def __del__(self):
+        """Destructor."""
         self.event_loop.close()
 
     def __enter__(self) -> "SyncEndpoint":
+        """Connect the endpoint."""
         self.connect()
         return self
 
@@ -30,6 +36,7 @@ def __exit__(
             exc_val: typing.Optional[BaseException],
             exc_tb: typing.Optional[types.TracebackType],
     ) -> None:
+        """Disconnect the endpoint."""
         self.disconnect()
 
     def connect(self):

eyepop/worker/worker_types.py

@@ -200,7 +200,7 @@ def FullForward(
     )
 
 
-class ComponentParams(BaseComponent):
+class ComponentParams(BaseModel):
     componentId: int
     values: dict[str, Any]
     model_config = ConfigDict(arbitrary_types_allowed=True, extra='forbid')

test_output.txt

@@ -0,0 +1,5 @@
+ERROR: usage: pytest [options] [file_or_dir] [file_or_dir] [...]
+pytest: error: unrecognized arguments: --timeout=300
+  inifile: /home/torsten/dev/projects/eyepop-ai/eyepop-sdk-python/pyproject.toml
+  rootdir: /home/torsten/dev/projects/eyepop-ai/eyepop-sdk-python
+