Refactor core logic and policy abstraction

Author: kyle65463

.gitignore

@@ -12,4 +12,5 @@ benchmarks/results/*
 temp/*
 *.log
 bin/*
-.venv/
+.venv/
+.env

benchmarks/benchmark.py

@@ -291,9 +291,9 @@ def get_vectorQ_answer(
         vectorQ_prompt = f"{task} {review_text}"
         latency_vectorq_logic: float = time.time()
         try:
-            is_cache_hit, cache_response, nn_response = self.vectorq.create(
+            is_cache_hit, cache_response, nn_response = self.vectorq.infer(
                 prompt=vectorQ_prompt,
-                output_format=output_format,
+                system_prompt=output_format,
                 benchmark=vectorQ_benchmark,
             )
         except Exception as e:

poetry.lock

@@ -22,6 +22,7 @@ dependencies = [
     "accelerate (>=1.6.0,<2.0.0)",
     "typing-extensions (>=4.13.2,<5.0.0)",
     "torchvision (>=0.22.0,<0.23.0)",
+    "statsmodels (>=0.14.4,<0.15.0)",
 ]
 
 
@@ -44,6 +45,7 @@ ruff = "^0.11.6"
 mypy = "^1.15.0"
 pre-commit = "^4.2.0"
 pytest = "^8.0.0"
+python-dotenv = "^1.1.0"
 
 
 [tool.ruff]

pyproject.toml

@@ -12,7 +12,7 @@
     LangChainInferenceEngine,
     OpenAIInferenceEngine,
 )
-from vectorq.main import VectorQ, VectorQBenchmark
+from vectorq.main import VectorQ
 
 # Embedding engines
 from vectorq.vectorq_core.cache.embedding_engine import (
@@ -48,10 +48,17 @@
     StringComparisonSimilarityEvaluator,
 )
 
+# VectorQ Policies
+from vectorq.vectorq_policy import (
+    DynamicThresholdPolicy,
+    NoCachePolicy,
+    StaticThresholdPolicy,
+    VectorQPolicy,
+)
+
 __all__ = [
     # Main classes
     "VectorQ",
-    "VectorQBenchmark",
     "VectorQConfig",
     # Inference engines
     "InferenceEngine",
@@ -74,8 +81,12 @@
     # Eviction policies
     "EvictionPolicy",
     "LRUEvictionPolicy",
-    "NoEvictionPolicy",
     # Embedding metadata storage
     "EmbeddingMetadataStorage",
     "InMemoryEmbeddingMetadataStorage",
+    # VectorQ Policies
+    "VectorQPolicy",
+    "DynamicThresholdPolicy",
+    "StaticThresholdPolicy",
+    "NoCachePolicy",
 ]

vectorq/__init__.py

@@ -1,19 +1,21 @@
 from typing import Optional
 
 from vectorq.inference_engine.inference_engine import InferenceEngine
+from vectorq.inference_engine.strategies.open_ai import OpenAIInferenceEngine
+from vectorq.vectorq_core.cache.embedding_engine import OpenAIEmbeddingEngine
 from vectorq.vectorq_core.cache.embedding_engine.embedding_engine import EmbeddingEngine
 from vectorq.vectorq_core.cache.embedding_store.embedding_metadata_storage.embedding_metadata_storage import (
     EmbeddingMetadataStorage,
 )
-from vectorq.vectorq_core.cache.embedding_store.vector_db import VectorDB
-from vectorq.vectorq_core.cache.eviction_policy.eviction_policy import EvictionPolicy
-from vectorq.vectorq_core.similarity_evaluator.similarity_evaluator import (
-    SimilarityEvaluator,
+from vectorq.vectorq_core.cache.embedding_store.embedding_metadata_storage.strategies.in_memory import (
+    InMemoryEmbeddingMetadataStorage,
 )
-from vectorq.vectorq_core.similarity_evaluator.strategies.string_comparison import (
-    StringComparisonSimilarityEvaluator,
+from vectorq.vectorq_core.cache.embedding_store.vector_db import VectorDB
+from vectorq.vectorq_core.cache.embedding_store.vector_db.strategies.hnsw_lib import (
+    HNSWLibVectorDB,
 )
-from vectorq.vectorq_core.vectorq_policy.vectorq_policy import VectorQPolicy
+from vectorq.vectorq_core.cache.eviction_policy.eviction_policy import EvictionPolicy
+from vectorq.vectorq_core.cache.eviction_policy.strategies.lru import LRUEvictionPolicy
 
 
 class VectorQConfig:
@@ -24,29 +26,16 @@ class VectorQConfig:
 
     def __init__(
         self,
-        accuracy_target: float = 0.8,
-        enable_cache: bool = True,
-        rnd_num_ub: float = 1.0,
-        is_static_threshold: bool = False,
-        static_threshold: float = 0.0,
-        inference_engine: Optional[InferenceEngine] = None,
-        embedding_engine: Optional[EmbeddingEngine] = None,
-        vector_db: Optional[VectorDB] = None,
-        similarity_evaluator: SimilarityEvaluator = StringComparisonSimilarityEvaluator(),
-        eviction_policy: Optional[EvictionPolicy] = None,
-        embedding_metadata_storage: Optional[EmbeddingMetadataStorage] = None,
-        vectorq_policy: Optional[VectorQPolicy] = None,
+        inference_engine: InferenceEngine = OpenAIInferenceEngine(),
+        embedding_engine: EmbeddingEngine = OpenAIEmbeddingEngine(),
+        vector_db: VectorDB = HNSWLibVectorDB(),
+        embedding_metadata_storage: EmbeddingMetadataStorage = InMemoryEmbeddingMetadataStorage(),
+        eviction_policy: EvictionPolicy = LRUEvictionPolicy(),
+        system_prompt: Optional[str] = None,
     ):
-        self.accuracy_target: float = accuracy_target
-        self.enable_cache: bool = enable_cache
-        self.rnd_num_ub: float = rnd_num_ub
-        self.is_static_threshold: bool = is_static_threshold
-        self.static_threshold: float = static_threshold
-
         self.inference_engine = inference_engine
         self.embedding_engine = embedding_engine
         self.vector_db = vector_db
-        self.similarity_evaluator = similarity_evaluator
         self.eviction_policy = eviction_policy
         self.embedding_metadata_storage = embedding_metadata_storage
-        self.vectorq_policy = vectorq_policy
+        self.system_prompt = system_prompt

vectorq/config.py

@@ -7,7 +7,7 @@ class InferenceEngine(ABC):
     """
 
     @abstractmethod
-    def create(self, prompt: str, output_format: str = None) -> str:
+    def create(self, prompt: str, system_prompt: str = None) -> str:
         """
         prompt: str - The prompt to create an answer for
         output_format: str - The optional output format to use for the response

vectorq/inference_engine/inference_engine.py

@@ -0,0 +1,19 @@
+from typing import override
+
+from vectorq.inference_engine.inference_engine import InferenceEngine
+
+
+class BenchmarkInferenceEngine(InferenceEngine):
+    """
+    An inference engine implementation that returns pre-computed responses for given prompts.
+    It is used for benchmarking purposes.
+    """
+
+    def set_next_response(self, response: str):
+        self.next_response = response
+
+    @override
+    def create(self, prompt: str, system_prompt: str = None) -> str:
+        if self.next_response is None:
+            raise ValueError("No next response set")
+        return self.next_response

vectorq/inference_engine/strategies/benchmark.py

@@ -49,11 +49,11 @@ def __init__(self, provider: str, model_name: str, temperature: float = 1):
         except Exception as e:
             raise Exception(f"Error initializing LangChain model: {e}")
 
-    def create(self, prompt: str, output_format: str = None) -> str:
+    def create(self, prompt: str, system_prompt: str = None) -> str:
         try:
             messages = []
-            if output_format:
-                messages.append(SystemMessage(content=output_format))
+            if system_prompt:
+                messages.append(SystemMessage(content=system_prompt))
             messages.append(HumanMessage(content=prompt))
 
             response = self.chat_model(messages)

vectorq/inference_engine/strategies/lang_chain.py

@@ -10,11 +10,11 @@ def __init__(self, model_name: str, temperature: float = 1):
         self.temperature = temperature
         self.client = OpenAIClient()
 
-    def create(self, prompt: str, output_format: str = None) -> str:
+    def create(self, prompt: str, system_prompt: str = None) -> str:
         try:
             messages = []
-            if output_format:
-                messages.append({"role": "system", "content": output_format})
+            if system_prompt:
+                messages.append({"role": "system", "content": system_prompt})
             messages.append({"role": "user", "content": prompt})
             completion = self.client.chat.completions.create(
                 model=self.model_name,