Update litserve example

Author: chiragjn

deploy-model-with-fastapi/server.py

@@ -6,10 +6,17 @@
 import pandas as pd
 from fastapi import FastAPI
 
+
+def _get_model_dir():
+    if "MODEL_DIR" not in os.environ:
+        raise Exception(
+            "MODEL_DIR environment variable is not set. Please set it to the directory containing the model."
+        )
+    return os.environ["MODEL_DIR"]
+
+
 model = None
-THIS_DIR = os.path.dirname(os.path.abspath(__file__))
-MODEL_DIR = os.getenv("MODEL_DIR", THIS_DIR)
-MODEL_PATH = os.path.join(MODEL_DIR, "iris_classifier.joblib")
+MODEL_PATH = os.path.join(_get_model_dir(), "iris_classifier.joblib")
 
 
 def load_model():
@@ -33,7 +40,9 @@ async def health() -> Dict[str, bool]:
 
 
 @app.post("/predict")
-def predict(sepal_length: float, sepal_width: float, petal_length: float, petal_width: float):
+def predict(
+    sepal_length: float, sepal_width: float, petal_length: float, petal_width: float
+):
     global model
     class_names = ["setosa", "versicolor", "virginica"]
     data = dict(

deploy-model-with-litserve/README.md

@@ -1,21 +1,24 @@
 # Deploy Whisper Model with Litserve
 
-## Setup
+---
+
+###  Install requirements
 
 ```bash
 pip install -r requirements.txt
 ```
 
-## Deploy
+### Start the server
 
 ```bash
-python deploy.py --workspace-fqn ... --host ... --port ...
+export MODEL_DIR="Systran/faster-whisper-tiny"
+python whisper_server.py
 ```
 
-## Test
+### Example inference call
 
 ```bash
-curl -X POST http://<endpoint>/predict -F "request=@./audio.mp3"
+curl -X POST http://0.0.0.0:8000/predict -F "request=@./audio.mp3"
 ```
 
 You should get the following response:

deploy-model-with-litserve/client.py

@@ -0,0 +1,7 @@
+# This file is auto-generated by LitServe.
+# Disable auto-generation by setting `generate_client_file=False` in `LitServer.run()`.
+
+import requests
+
+response = requests.post("http://127.0.0.1:8000/predict", json={"input": 4.0})
+print(f"Status: {response.status_code}\nResponse:\n {response.text}")

deploy-model-with-litserve/deploy.py

@@ -1,12 +1,21 @@
+import os
 import litserve as ls
 from fastapi import UploadFile
 from faster_whisper import WhisperModel
 
+def _get_model_dir():
+    if "MODEL_DIR" not in os.environ:
+        raise Exception(
+            "MODEL_DIR environment variable is not set. Please set it to the directory containing the model."
+        )
+    return os.environ["MODEL_DIR"]
+
+MODEL_DIR = _get_model_dir()
 
 class WhisperLitAPI(ls.LitAPI):
     def setup(self, device):
         # Load the OpenAI Whisper model. You can specify other models like "base", "small", etc.
-        self.model = WhisperModel("tiny", device="cpu")
+        self.model = WhisperModel(MODEL_DIR, device="cpu")
 
     def decode_request(self, request: UploadFile):
         # Assuming the request sends the path to the audio file
@@ -28,5 +37,11 @@ def encode_response(self, output):
 
 if __name__ == "__main__":
     api = WhisperLitAPI()
-    server = ls.LitServer(api, fast_queue=True, accelerator="cpu", timeout=1000, workers_per_device=1)
+    server = ls.LitServer(
+        api, 
+        fast_queue=True, 
+        accelerator="cpu", 
+        timeout=1000, 
+        workers_per_device=1
+    )
     server.run(port=8000)