IID Baseline

.gitignore

@@ -13,4 +13,5 @@ temp/*
 *.log
 bin/*
 .venv/
-.env
+.env
+*.png

benchmarks/_plotter_combined.py

@@ -100,10 +100,14 @@ def generate_combined_plots(
     vectorq_global_data_frames: Dict[float, pd.DataFrame] = {}
     for global_file_path in vectorq_global_files:
         with open(global_file_path, "r") as f:
-            data: Any = json.load(f)
-            dataframe, _ = convert_to_dataframe_from_json_file(data)
-            delta: float = data["config"]["delta"]
-            vectorq_global_data_frames[delta] = dataframe
+            try:
+                data: Any = json.load(f)
+                dataframe, _ = convert_to_dataframe_from_json_file(data)
+                delta: float = data["config"]["delta"]
+                vectorq_global_data_frames[delta] = dataframe
+            except Exception as e:
+                print(f"Error loading {global_file_path}: {e}")
+                continue
 
     __plot_roc(
         static_data_frames=static_data_frames,
@@ -184,14 +188,15 @@ def __plot_roc(
             static_tpr_values,
             "o-",
             color="blue",
-            linewidth=2,
+            linewidth=3,
             label="GPTCache",
-            markersize=8,
+            markersize=10,
         )
 
         for i, threshold in enumerate(static_thresholds):
             if i == 0 or i == len(static_thresholds) - 1:
-                label = f"{threshold:.2f}"
+                # label = f"{threshold:.2f}"
+                label = None
             else:
                 label = None
             plt.annotate(
@@ -226,14 +231,15 @@ def __plot_roc(
             vectorq_local_tpr_values,
             "o-",
             color="green",
-            linewidth=2,
-            label="VectorQ (Local)",
-            markersize=8,
+            linewidth=3,
+            label="vCache",
+            markersize=10,
         )
 
         for i, _ in enumerate(vectorq_local_tpr_values):
             if i == 0 or i == len(vectorq_local_deltas) - 1:
-                label = f"{vectorq_local_deltas[i]:.2f}"
+                # label = f"{vectorq_local_deltas[i]:.2f}"
+                label = None
             else:
                 label = None
             plt.annotate(
@@ -265,14 +271,15 @@ def __plot_roc(
             vectorq_global_tpr_values,
             "o-",
             color="red",
-            linewidth=2,
-            label="VectorQ (Global)",
-            markersize=8,
+            linewidth=3,
+            label="vCache (Ablation)",
+            markersize=10,
         )
 
         for i, delta in enumerate(vectorq_global_deltas):
             if i == 0 or i == len(vectorq_global_deltas) - 1:
-                label = f"{delta:.2f}"
+                # label = f"{delta:.2f}"
+                label = None
             else:
                 label = None
             plt.annotate(
@@ -294,9 +301,17 @@ def __plot_roc(
 
     plt.xlim(0, 1)
     plt.ylim(0, 1)
+    yticks = plt.yticks()[0]
+    if yticks[0] == 0.0:
+        plt.yticks(yticks[1:])
+
+    plt.gca().spines["top"].set_linewidth(1)
+    plt.gca().spines["right"].set_linewidth(1)
+    plt.gca().spines["bottom"].set_linewidth(1)
+    plt.gca().spines["left"].set_linewidth(1)
 
     filename = results_dir + f"/roc_{timestamp}.pdf"
-    plt.savefig(filename, format="pdf", bbox_inches="tight")
+    plt.savefig(filename, format="pdf", transparent=True)
     plt.close()
 
 
@@ -618,14 +633,15 @@ def __plot_cache_hit_vs_error_rate(
             static_cache_hit_rates,
             "o-",
             color="blue",
-            linewidth=2,
+            linewidth=3,
             label="GPTCache",
-            markersize=8,
+            markersize=10,
         )
 
         for i, threshold in enumerate(static_thresholds):
             if i == 0 or i == len(static_thresholds) - 2:
-                label = f"{threshold:.2f}"
+                # label = f"{threshold:.2f}"
+                label = None
             else:
                 label = None
             plt.annotate(
@@ -662,17 +678,18 @@ def __plot_cache_hit_vs_error_rate(
             vectorq_local_cache_hit_rates,
             "o-",
             color="green",
-            linewidth=2,
-            label="VectorQ (Local)",
-            markersize=8,
+            linewidth=3,
+            label="vCache",
+            markersize=10,
         )
 
         for i, _ in enumerate(vectorq_local_error_rates):
             if i == 0:
                 continue
 
             if i == 0 or i == len(vectorq_local_deltas) - 1:
-                label = f"{vectorq_local_deltas[i]:.2f}"
+                # label = f"{vectorq_local_deltas[i]:.2f}"
+                label = None
             else:
                 label = None
             plt.annotate(
@@ -705,14 +722,15 @@ def __plot_cache_hit_vs_error_rate(
             vectorq_global_cache_hit_rates,
             "o-",
             color="red",
-            linewidth=2,
-            label="VectorQ (Global)",
-            markersize=8,
+            linewidth=3,
+            label="vCache (Ablation)",
+            markersize=10,
         )
 
         for i, delta in enumerate(vectorq_global_deltas):
             if i == 0 or i == len(vectorq_global_deltas) - 1:
-                label = f"{delta:.2f}"
+                # label = f"{delta:.2f}"
+                label = None
             else:
                 label = None
             plt.annotate(
@@ -729,8 +747,17 @@ def __plot_cache_hit_vs_error_rate(
     plt.grid(True, linestyle="--", alpha=0.7)
     plt.legend(loc="best", fontsize=font_size - 2)
     plt.tick_params(axis="both", labelsize=font_size - 2)
+
     plt.xlim(0, 0.3)
     plt.ylim(0, 1)
+    yticks = plt.yticks()[0]
+    if yticks[0] == 0.0:
+        plt.yticks(yticks[1:])
+
+    plt.gca().spines["top"].set_linewidth(1)
+    plt.gca().spines["right"].set_linewidth(1)
+    plt.gca().spines["bottom"].set_linewidth(1)
+    plt.gca().spines["left"].set_linewidth(1)
 
     filename = results_dir + f"/cache_hit_vs_error_rate_{timestamp}.pdf"
     plt.savefig(filename, format="pdf", bbox_inches="tight")

benchmarks/benchmark.py

@@ -44,6 +44,9 @@
 from vectorq.vectorq_policy.strategies.dynamic_local_threshold import (
     DynamicLocalThresholdPolicy,
 )
+from vectorq.vectorq_policy.strategies.iid_local_threshold import (
+    IIDLocalThresholdPolicy,
+)
 from vectorq.vectorq_policy.strategies.static_global_threshold import (
     StaticGlobalThresholdPolicy,
 )
@@ -64,7 +67,7 @@
 ########################################################################################################################
 
 # Benchmark Config
-MAX_SAMPLES: int = 5000
+MAX_SAMPLES: int = 15000
 CONFIDENCE_INTERVALS_ITERATIONS: int = 3
 IS_LLM_JUDGE_BENCHMARK: bool = False
 
@@ -104,7 +107,7 @@
     "ecommerce_dataset.json",
     "semantic_prompt_cache_benchmark.json",
 ]
-DATASETS_TO_EXCLUDE: List[str] = [DATASETS[0], DATASETS[2], DATASETS[3]]
+DATASETS_TO_EXCLUDE: List[str] = [DATASETS[1], DATASETS[2]]
 
 embedding_models: List[Tuple[str, str, str, int]] = [
     EMBEDDING_MODEL_1,
@@ -123,9 +126,16 @@
 
 # VectorQ Config
 MAX_VECTOR_DB_CAPACITY: int = 100000
-PLOT_FONT_SIZE: int = 24
-
-SYSTEM_TYPES: List[str] = ["static", "dynamic_local", "dynamic_global", "all"]
+PLOT_FONT_SIZE: int = 32
+
+SYSTEM_TYPES: List[str] = [
+    "static",
+    "dynamic_local",
+    "dynamic_global",
+    "iid_local",
+    "all",
+    "just_plot",
+]
 SYSTEM_TYPE: str = SYSTEM_TYPES[3]
 
 
@@ -548,7 +558,36 @@ def main():
                                 threshold=-1,
                             )
 
-                # Baseline 3) Static thresholds
+                # Baseline 3) IID Local thresholds
+                if SYSTEM_TYPE in ["iid_local", "all"]:
+                    for delta in deltas:
+                        for i in range(0, CONFIDENCE_INTERVALS_ITERATIONS):
+                            path = os.path.join(
+                                results_dir,
+                                dataset,
+                                embedding_model[1],
+                                llm_model[1],
+                                f"iid_local_{delta}_run_{i + 1}",
+                            )
+                            if os.path.exists(path) and os.listdir(path):
+                                continue
+
+                            logging.info(
+                                f"Using IID local threshold with delta: {delta}. Run {i + 1} of {CONFIDENCE_INTERVALS_ITERATIONS}"
+                            )
+
+                            __run_baseline(
+                                vectorq_policy=IIDLocalThresholdPolicy(delta=delta),
+                                path=path,
+                                dataset_file=dataset_file,
+                                embedding_model=embedding_model,
+                                llm_model=llm_model,
+                                timestamp=timestamp,
+                                delta=delta,
+                                threshold=-1,
+                            )
+
+                # Baseline 4) Static thresholds
                 if SYSTEM_TYPE in ["static", "all"]:
                     for threshold in static_thresholds:
                         path = os.path.join(
@@ -576,7 +615,7 @@ def main():
                             threshold=threshold,
                         )
 
-                if SYSTEM_TYPE == "all":
+                if SYSTEM_TYPE == "all" or SYSTEM_TYPE == "just_plot":
                     generate_combined_plots(
                         dataset=dataset,
                         embedding_model_name=embedding_model[1],

vectorq/__init__.py

@@ -51,6 +51,7 @@
 from vectorq.vectorq_policy import (
     DynamicGlobalThresholdPolicy,
     DynamicLocalThresholdPolicy,
+    IIDLocalThresholdPolicy,
     NoCachePolicy,
     StaticGlobalThresholdPolicy,
     VectorQPolicy,
@@ -89,4 +90,5 @@
     "DynamicGlobalThresholdPolicy",
     "StaticGlobalThresholdPolicy",
     "NoCachePolicy",
+    "IIDLocalThresholdPolicy",
 ]

vectorq/vectorq_core/cache/embedding_store/embedding_metadata_storage/embedding_metadata_obj.py

@@ -26,8 +26,6 @@ def __init__(
         self.t_hat: float = None
         self.t_prime: float = None
         self.var_t: float = None
-        self.observations.append((0.0, 0))
-        self.observations.append((1.0, 1))
         self.gamma: float = None
         self.t_hat: float = None
         ##################################################

vectorq/vectorq_policy/__init__.py

@@ -4,6 +4,9 @@
 from vectorq.vectorq_policy.strategies.dynamic_local_threshold import (
     DynamicLocalThresholdPolicy,
 )
+from vectorq.vectorq_policy.strategies.iid_local_threshold import (
+    IIDLocalThresholdPolicy,
+)
 from vectorq.vectorq_policy.strategies.no_cache import NoCachePolicy
 from vectorq.vectorq_policy.strategies.static_global_threshold import (
     StaticGlobalThresholdPolicy,
@@ -15,5 +18,6 @@
     "StaticGlobalThresholdPolicy",
     "DynamicLocalThresholdPolicy",
     "DynamicGlobalThresholdPolicy",
+    "IIDLocalThresholdPolicy",
     "NoCachePolicy",
 ]

vectorq/vectorq_policy/strategies/dynamic_global_threshold.py

@@ -33,7 +33,7 @@ def __init__(
         Args
             delta: float - The delta value to use
         """
-        self.bayesian = _Bayesian(delta=delta)
+        self.bayesian = _Algorithm(delta=delta)
         self.similarity_evaluator: SimilarityEvaluator = None
         self.inference_engine: InferenceEngine = None
         self.cache: Cache = None
@@ -107,7 +107,7 @@ class _Action(Enum):
     EXPLOIT = "exploit"
 
 
-class _Bayesian:
+class _Algorithm:
     def __init__(self, delta: float):
         self.delta: float = delta
         self.P_c: float = 1.0 - self.delta

vectorq/vectorq_policy/strategies/dynamic_local_threshold.py

@@ -16,7 +16,9 @@
     EmbeddingMetadataObj,
 )
 from vectorq.vectorq_core.cache.embedding_store.embedding_store import EmbeddingStore
-from vectorq.vectorq_core.similarity_evaluator import SimilarityEvaluator
+from vectorq.vectorq_core.similarity_evaluator import (
+    SimilarityEvaluator,
+)
 from vectorq.vectorq_policy.vectorq_policy import VectorQPolicy
 
 
@@ -30,7 +32,7 @@ def __init__(self, delta: float = 0.01):
         Args
             delta: float - The delta value to use
         """
-        self.bayesian = _Bayesian(delta=delta)
+        self.bayesian = _Algorithm(delta=delta)
         self.similarity_evaluator: SimilarityEvaluator = None
         self.inference_engine: InferenceEngine = None
         self.cache: Cache = None
@@ -104,7 +106,7 @@ class _Action(Enum):
     EXPLOIT = "exploit"
 
 
-class _Bayesian:
+class _Algorithm:
     def __init__(self, delta: float):
         self.delta: float = delta
         self.P_c: float = 1.0 - self.delta