update rls performance monitoring

Author: giobbu

README.md

@@ -15,7 +15,7 @@ The CUSUM repository contains multiple change point detectors for sequential ana
 1. [CUSUM Detectors](#1-cusum-detectors)
 2. [Getting Started](#2-getting-started)
 3. [Documentation](#3-documentation)
-4. [Example: ML Model Performance Monitoring](#4-example-ml-model-performance-monitoring)
+4. [Examples](#4-examples)
 5. [License](#5-license)
 
 ## 0. Overview
@@ -53,17 +53,19 @@ uv sync
 
 ## 3. Documentation
 
-Documentation is available at [CUSUM Documentation](https://CUSUM.readthedocs.io/en/latest/)
+Documentation is available at [CUSUM Docs](https://CUSUM.readthedocs.io/en/latest/)
 
 
-## 4. Example: ML Model Performance Monitoring
+## 4. Examples: 
+
+### a) ML Model Performance Monitoring with Observed Labels
 
 Performance Monitoring of an instance-based linear learning model applying the CUSUM algorithm.
 
 At each time step: 
 
 * Generate a prediction with recursive least squares (RLS) model;
-* Acquire the true observed value (if you are lucky);
+* Acquire the true observed value;
 * Compute residual;
 * Apply the CUSUM detector on the residuals to identify potential change points;
 * Update the model parameters with the new data instance.

docs/_static/images/monitoring.gif

@@ -9,10 +9,16 @@
     "import sys\n",
     "import os\n",
     "import numpy as np\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()\n",
+    "sys.path.append(os.getenv(\"PATH_CUSUM\"))\n",
     "from source.model.incremental import RecursiveLeastSquares\n",
     "from source.detector.cusum import CUSUM_Detector\n",
     "import matplotlib.pyplot as plt\n",
-    "from matplotlib.animation import FuncAnimation"
+    "from matplotlib.animation import FuncAnimation\n",
+    "\n",
+    "# set random seed for reproducibility\n",
+    "np.random.seed(42)"
    ]
   },
   {
@@ -21,7 +27,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "n = 600\n",
+    "n = 300\n",
     "\n",
     "# Create time series components\n",
     "t = np.arange(n)\n",
@@ -52,7 +58,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "values[300:500] += 50  # Introduce a change point\n",
+    "values[150:] += 50  # Introduce a change point\n",
     "\n",
     "# plot the time series\n",
     "plt.figure(figsize=(15, 4))\n",
@@ -77,23 +83,25 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Set number of lags\n",
-    "num_lags = 3\n",
-    "\n",
     "# Initialize model\n",
-    "model = RecursiveLeastSquares(num_variables=num_lags, forgetting_factor=0.99, initial_delta=50)\n",
-    "\n",
-    "warmup_period = 100\n",
+    "num_lags = 5\n",
+    "model = RecursiveLeastSquares(num_variables=num_lags, \n",
+    "                              forgetting_factor=0.999, \n",
+    "                              initial_delta=1.0)\n",
     "\n",
     "# Detect change points using CUSUM Detector\n",
-    "cusum_detector = CUSUM_Detector(warmup_period=warmup_period, delta=3, threshold=5)\n",
+    "warmup_period = 50\n",
+    "cusum_detector = CUSUM_Detector(warmup_period=warmup_period, \n",
+    "                                delta=1.5, \n",
+    "                                threshold=4.0)\n",
+    "\n",
     "\n",
     "# Store predictions and observed values\n",
     "list_predictions = []\n",
     "list_observed = []\n",
     "list_residuals = []\n",
     "\n",
-    "list_pos_changes = []\n",
+    "list_acc_changes = []\n",
     "list_change_points = []\n",
     "list_index_changes = []\n",
     "list_output_detected = []\n",
@@ -104,7 +112,7 @@
     "fig.tight_layout(pad=5.0)\n",
     "\n",
     "# Animation function\n",
-    "def animate(i):\n",
+    "def monitoring(i):\n",
     "\n",
     "    global output_detected, list_change_points, list_output_detected, list_residuals\n",
     "    \n",
@@ -125,18 +133,18 @@
     "\n",
     "        # Store prediction and observed values\n",
     "        list_predictions.append(y_pred)\n",
-    "        list_observed.append(y[0])\n",
+    "        list_observed.append(float(y[0][0]))\n",
     "\n",
     "        # model params update\n",
     "        model.update(X, y)\n",
     "\n",
     "        # Compute residuals\n",
     "        residuals = np.abs(y - y_pred)\n",
-    "        list_residuals.append(residuals[0])\n",
+    "        list_residuals.append(residuals[0][0])\n",
     "\n",
     "        # Detect change points\n",
     "        output_detected = cusum_detector.detection(residuals[0])\n",
-    "        list_pos_changes.append(output_detected[0][0])\n",
+    "        list_acc_changes.append(float(output_detected[0][0]))\n",
     "        list_change_points.append(output_detected[-1])\n",
     "        list_output_detected.append(output_detected)\n",
     "\n",
@@ -146,35 +154,69 @@
     "        change_ax.clear()\n",
     "\n",
     "        # Plot predictions vs observed values\n",
-    "        prediction_ax.plot(list_observed, label='Obs', color='blue')\n",
-    "        prediction_ax.plot(list_predictions, label='Pred', color='orange')\n",
+    "        prediction_ax.plot(list_observed, label='Observation', color='orange')\n",
+    "        prediction_ax.plot(list_predictions, label='Prediction', color='blue', marker='o', markersize=4)\n",
+    "        prediction_ax.fill_between(range(len(list_predictions)),\n",
+    "                                   list_predictions,\n",
+    "                                   list_observed,\n",
+    "                                   color='lightgray', \n",
+    "                                   alpha=0.7)\n",
     "        for i, is_cp in enumerate(list_change_points):\n",
     "            if is_cp:\n",
     "                prediction_ax.axvline(x=i, color='red', linestyle='--', alpha=0.5)\n",
-    "        if i >= warmup_period:\n",
-    "            prediction_ax.axvline(x=warmup_period, color='gray', label='End WarmUp', linestyle='--')\n",
     "        prediction_ax.plot()\n",
     "        prediction_ax.legend()\n",
-    "        prediction_ax.set_title(f'Change Points at idx {100} and {500}\\n RLS Predictions vs Observed Values')\n",
+    "        prediction_ax.set_title(f'Predictions vs Observations')\n",
+    "        prediction_ax.set_xlabel('Time Step')\n",
+    "        prediction_ax.set_ylabel('Prediction Value')\n",
+    "        prediction_ax.grid()\n",
     "        \n",
     "        # Plot residuals\n",
-    "        residual_ax.plot(list_residuals, label='Residuals', color='red')\n",
+    "        residual_ax.plot(list_residuals, label='Residuals', color='red', marker='o', markersize=4) \n",
+    "        residual_ax.fill_between(range(len(list_residuals)),\n",
+    "                                 0,\n",
+    "                                 list_residuals,\n",
+    "                                 color='red', \n",
+    "                                 alpha=0.1)                     \n",
     "        residual_ax.legend()\n",
     "        residual_ax.set_title('Residuals')\n",
+    "        residual_ax.set_xlabel('Time Step')\n",
+    "        residual_ax.set_ylabel('Residual Value')\n",
+    "        residual_ax.grid()\n",
     "\n",
     "        # Plot detected change points\n",
-    "        change_ax.plot(list_pos_changes, label='Performance Cumulative Changes', color='green')\n",
-    "        change_ax.axhline(y=cusum_detector.threshold, color='r', linestyle='--')\n",
+    "        change_ax.plot(list_acc_changes, label='Cumulative Change', color='green', marker='o', markersize=4)\n",
+    "        change_ax.fill_between(range(len(list_acc_changes)),\n",
+    "                                0,\n",
+    "                                list_acc_changes,\n",
+    "                                color='green', \n",
+    "                                alpha=0.5)\n",
+    "        change_ax.axhline(y=cusum_detector.threshold, color='r', linestyle='--', label='CUSUM Threshold')\n",
     "        change_ax.legend()\n",
     "        change_ax.set_title('Detected Change Points')\n",
+    "        change_ax.set_xlabel('Time Step')\n",
+    "        change_ax.set_ylabel('Cumulative Change Value')\n",
+    "        change_ax.grid()\n",
     "\n",
     "# Create the animation\n",
-    "ani = FuncAnimation(fig, animate, frames=len(values) - num_lags - 1, repeat=False)  #\n",
+    "ani = FuncAnimation(fig, monitoring, frames=len(values) - num_lags - 1, repeat=False)  #\n",
     "\n",
     "# Save the animation\n",
     "ani.save('monitoring.gif', writer='pillow', fps=100)\n",
     "plt.close(fig)  # Prevent the figure from displaying"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import IPython.display as display\n",
+    "\n",
+    "# Display the saved GIF\n",
+    "display.Image(filename='monitoring.gif')"
+   ]
   }
  ],
  "metadata": {