Added Similarity Evaluator To Config

Author: luis-gasparschroeder

benchmarks/benchmark.py

@@ -32,6 +32,12 @@
     HNSWLibVectorDB,
     SimilarityMetricType,
 )
+from vectorq.vectorq_core.similarity_evaluator.strategies.llm_comparison import (
+    LLMComparisonSimilarityEvaluator,
+)
+from vectorq.vectorq_core.similarity_evaluator.strategies.string_comparison import (
+    StringComparisonSimilarityEvaluator,
+)
 from vectorq.vectorq_policy.strategies.dynamic_global_threshold import (
     DynamicGlobalThresholdPolicy,
 )
@@ -58,8 +64,10 @@
 ########################################################################################################################
 
 # Benchmark Config
-MAX_SAMPLES: int = 15000
+MAX_SAMPLES: int = 5000
 CONFIDENCE_INTERVALS_ITERATIONS: int = 3
+IS_LLM_JUDGE_BENCHMARK: bool = False
+
 EMBEDDING_MODEL_1 = (
     "embedding_1",
     "GteLargeENv1_5",
@@ -96,7 +104,7 @@
     "ecommerce_dataset.json",
     "semantic_prompt_cache_benchmark.json",
 ]
-DATASETS_TO_EXCLUDE: List[str] = [DATASETS[1], DATASETS[2]]
+DATASETS_TO_EXCLUDE: List[str] = [DATASETS[0], DATASETS[2], DATASETS[3]]
 
 embedding_models: List[Tuple[str, str, str, int]] = [
     EMBEDDING_MODEL_1,
@@ -410,6 +418,11 @@ def __run_baseline(
     delta: float,
     threshold: float,
 ):
+    if IS_LLM_JUDGE_BENCHMARK:
+        similarity_evaluator = LLMComparisonSimilarityEvaluator()
+    else:
+        similarity_evaluator = StringComparisonSimilarityEvaluator()
+
     vectorq_config: VectorQConfig = VectorQConfig(
         inference_engine=BenchmarkInferenceEngine(),
         embedding_engine=BenchmarkEmbeddingEngine(),
@@ -418,6 +431,7 @@ def __run_baseline(
             max_capacity=MAX_VECTOR_DB_CAPACITY,
         ),
         embedding_metadata_storage=InMemoryEmbeddingMetadataStorage(),
+        similarity_evaluator=similarity_evaluator,
     )
     vectorQ: VectorQ = VectorQ(vectorq_config, vectorq_policy)
 

…vCache_Logo_For_Light_Background.png.png docs/vCache_Logo_For_Light_Background.pngdocs/vCache_Logo_For_Light_Background.png.png renamed to docs/vCache_Logo_For_Light_Background.png docs/vCache_Logo_For_Light_Background.png.png renamed to docs/vCache_Logo_For_Light_Background.png

@@ -8,7 +8,6 @@
     InMemoryEmbeddingMetadataStorage,
     LangChainEmbeddingEngine,
     OpenAIInferenceEngine,
-    StringComparisonSimilarityEvaluator,
     VectorQ,
     VectorQConfig,
 )
@@ -29,10 +28,7 @@ def create_default_config_and_policy():
         embedding_metadata_storage=InMemoryEmbeddingMetadataStorage(),
         system_prompt="Please answer in a single word with the first letter capitalized. Example: London",
     )
-    policy = DynamicLocalThresholdPolicy(
-        delta=0.05,
-        similarity_evaluator=StringComparisonSimilarityEvaluator(),
-    )
+    policy = DynamicLocalThresholdPolicy(delta=0.05)
     return config, policy
 
 

tests/integration/test_dynamic_threshold.py

@@ -18,6 +18,12 @@
 from vectorq.vectorq_core.cache.eviction_policy.strategies.no_eviction import (
     NoEvictionPolicy,
 )
+from vectorq.vectorq_core.similarity_evaluator.similarity_evaluator import (
+    SimilarityEvaluator,
+)
+from vectorq.vectorq_core.similarity_evaluator.strategies.string_comparison import (
+    StringComparisonSimilarityEvaluator,
+)
 
 
 class VectorQConfig:
@@ -33,11 +39,14 @@ def __init__(
         vector_db: VectorDB = HNSWLibVectorDB(),
         embedding_metadata_storage: EmbeddingMetadataStorage = InMemoryEmbeddingMetadataStorage(),
         eviction_policy: EvictionPolicy = NoEvictionPolicy(),
+        similarity_evaluator: SimilarityEvaluator = StringComparisonSimilarityEvaluator(),
         system_prompt: Optional[str] = None,
     ):
         self.inference_engine = inference_engine
         self.embedding_engine = embedding_engine
         self.vector_db = vector_db
         self.eviction_policy = eviction_policy
         self.embedding_metadata_storage = embedding_metadata_storage
+        self.similarity_evaluator = similarity_evaluator
+        self.similarity_evaluator.set_inference_engine(self.inference_engine)
         self.system_prompt = system_prompt

vectorq/config.py

@@ -1,7 +1,12 @@
 from abc import ABC, abstractmethod
 
+from vectorq.inference_engine import InferenceEngine
+
 
 class SimilarityEvaluator(ABC):
+    def __init__(self):
+        self.inference_engine: InferenceEngine = None
+
     @abstractmethod
     def answers_similar(self, a: str, b: str) -> bool:
         """
@@ -10,3 +15,6 @@ def answers_similar(self, a: str, b: str) -> bool:
         returns: bool - True if the answers are similar, False otherwise
         """
         pass
+
+    def set_inference_engine(self, inference_engine: InferenceEngine):
+        self.inference_engine = inference_engine

vectorq/vectorq_core/similarity_evaluator/similarity_evaluator.py

@@ -9,5 +9,7 @@ def __init__(self):
 
     def answers_similar(self, a: str, b: str) -> bool:
         # TODO
+        # @Alex: You can access the inference engine via:
+        # self.inference_engine
         print("TODO: Embedding-based Answer similarity check not implemented")
         return False

vectorq/vectorq_core/similarity_evaluator/strategies/llm_comparison.py

@@ -10,22 +10,19 @@
 from typing_extensions import override
 
 from vectorq.config import VectorQConfig
+from vectorq.inference_engine import InferenceEngine
 from vectorq.vectorq_core.cache.cache import Cache
 from vectorq.vectorq_core.cache.embedding_store.embedding_metadata_storage.embedding_metadata_obj import (
     EmbeddingMetadataObj,
 )
 from vectorq.vectorq_core.cache.embedding_store.embedding_store import EmbeddingStore
-from vectorq.vectorq_core.similarity_evaluator import (
-    SimilarityEvaluator,
-    StringComparisonSimilarityEvaluator,
-)
+from vectorq.vectorq_core.similarity_evaluator import SimilarityEvaluator
 from vectorq.vectorq_policy.vectorq_policy import VectorQPolicy
 
 
 class DynamicGlobalThresholdPolicy(VectorQPolicy):
     def __init__(
         self,
-        similarity_evaluator: SimilarityEvaluator = StringComparisonSimilarityEvaluator(),
         delta: float = 0.01,
     ):
         """
@@ -34,16 +31,16 @@ def __init__(
         This is suboptimal in cases when the embeddings cannot seperate correct from incorrect responses.
 
         Args
-            similarity_evaluator: SimilarityEvaluator - The similarity evaluator to use
             delta: float - The delta value to use
         """
-        self.similarity_evaluator = similarity_evaluator
         self.bayesian = _Bayesian(delta=delta)
-        self.inference_engine = None
-        self.cache = None
+        self.similarity_evaluator: SimilarityEvaluator = None
+        self.inference_engine: InferenceEngine = None
+        self.cache: Cache = None
 
     @override
     def setup(self, config: VectorQConfig):
+        self.similarity_evaluator = config.similarity_evaluator
         self.inference_engine = config.inference_engine
         self.cache = Cache(
             embedding_engine=config.embedding_engine,

vectorq/vectorq_policy/strategies/dynamic_global_threshold.py

@@ -10,40 +10,34 @@
 from typing_extensions import override
 
 from vectorq.config import VectorQConfig
+from vectorq.inference_engine import InferenceEngine
 from vectorq.vectorq_core.cache.cache import Cache
 from vectorq.vectorq_core.cache.embedding_store.embedding_metadata_storage.embedding_metadata_obj import (
     EmbeddingMetadataObj,
 )
 from vectorq.vectorq_core.cache.embedding_store.embedding_store import EmbeddingStore
-from vectorq.vectorq_core.similarity_evaluator import (
-    SimilarityEvaluator,
-    StringComparisonSimilarityEvaluator,
-)
+from vectorq.vectorq_core.similarity_evaluator import SimilarityEvaluator
 from vectorq.vectorq_policy.vectorq_policy import VectorQPolicy
 
 
 class DynamicLocalThresholdPolicy(VectorQPolicy):
-    def __init__(
-        self,
-        similarity_evaluator: SimilarityEvaluator = StringComparisonSimilarityEvaluator(),
-        delta: float = 0.01,
-    ):
+    def __init__(self, delta: float = 0.01):
         """
         This policy uses the VectorQ algorithm to compute the optimal threshold for each
         embedding in the cache.
         Each threshold is used to determine if a response is a cache hit.
 
         Args
-            similarity_evaluator: SimilarityEvaluator - The similarity evaluator to use
             delta: float - The delta value to use
         """
-        self.similarity_evaluator = similarity_evaluator
         self.bayesian = _Bayesian(delta=delta)
-        self.inference_engine = None
-        self.cache = None
+        self.similarity_evaluator: SimilarityEvaluator = None
+        self.inference_engine: InferenceEngine = None
+        self.cache: Cache = None
 
     @override
     def setup(self, config: VectorQConfig):
+        self.similarity_evaluator = config.similarity_evaluator
         self.inference_engine = config.inference_engine
         self.cache = Cache(
             embedding_engine=config.embedding_engine,