feat: add observability and smoke tests

Author: haasonsaas

.github/workflows/smoke-tests.yml

@@ -0,0 +1,18 @@
+name: Smoke Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  smoke-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run smoke tests
+        run: |
+          COMPOSE_FILE=docker-compose.yml ./scripts/smoke-tests.sh

Makefile

@@ -12,6 +12,10 @@ build:
 
 test:
 	go test ./...
+	python3 -m pytest
+
+smoke:
+	COMPOSE_FILE=docker-compose.yml ./scripts/smoke-tests.sh
 
 # Formatting targets
 format: format-go format-python

app/main.py

@@ -1,26 +1,50 @@
-from flask import Flask, request, jsonify
+import logging
+import time
+from contextlib import contextmanager
+
+from flask import Flask, jsonify, request
+
+from app.telemetry import setup as telemetry_setup
+
 
 app = Flask(__name__)
+telemetry_setup(app, service_name="keep-app")
+
+
+@contextmanager
+def record_route_metrics(route: str):
+    start = time.perf_counter()
+    try:
+        yield
+    finally:
+        duration_ms = (time.perf_counter() - start) * 1000
+        logging.getLogger(__name__).info(
+            "route completed",
+            extra={"route": route, "duration_ms": round(duration_ms, 2)},
+        )
 
 
 @app.route("/health")
 def health():
-    return {"status": "ok"}
+    with record_route_metrics("health"):
+        return {"status": "ok"}
 
 
 @app.route("/")
 def index():
-    cert_subject = request.headers.get("X-Client-Subject", "unknown")
-    if cert_subject.startswith("Subject="):
-        cert_subject = cert_subject.replace("Subject=", "", 1)
-    device_id = request.headers.get("X-Device-ID", "unknown")
-    return (
-        f"Hello from keep protected app!\n"
-        f"Client cert subject: {cert_subject}\n"
-        f"Device ID: {device_id}\n"
-    )
+    with record_route_metrics("index"):
+        cert_subject = request.headers.get("X-Client-Subject", "unknown") or "unknown"
+        if cert_subject.startswith("Subject="):
+            cert_subject = cert_subject.replace("Subject=", "", 1)
+        device_id = request.headers.get("X-Device-ID", "unknown") or "unknown"
+        return (
+            f"Hello from keep protected app!\n"
+            f"Client cert subject: {cert_subject}\n"
+            f"Device ID: {device_id}\n"
+        )
 
 
 @app.route("/step-up", methods=["POST"])
 def step_up():
-    return jsonify({"status": "step-up required"}), 202
+    with record_route_metrics("step_up"):
+        return jsonify({"status": "step-up required"}), 202

app/requirements.txt

@@ -1,3 +1,7 @@
 Flask==3.0.3
 gunicorn==21.2.0
 requests==2.32.3
+opentelemetry-sdk==1.26.0
+opentelemetry-exporter-otlp-proto-http==1.26.0
+opentelemetry-instrumentation-flask==0.47b0
+prometheus-client==0.20.0

app/telemetry.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import logging
+import os
+from typing import Optional
+
+from opentelemetry import metrics, trace
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+_logger = logging.getLogger(__name__)
+
+
+def _resource(service_name: str, environment: str) -> Resource:
+    return Resource.create({
+        "service.name": service_name,
+        "deployment.environment": environment,
+    })
+
+
+def init_tracing(service_name: str, environment: str, endpoint: str, insecure: bool) -> None:
+    exporter = OTLPSpanExporter(endpoint=endpoint or None, insecure=insecure)
+    tracer_provider = TracerProvider(resource=_resource(service_name, environment))
+    tracer_provider.add_span_processor(BatchSpanProcessor(exporter))
+    trace.set_tracer_provider(tracer_provider)
+
+
+def init_metrics(service_name: str, environment: str, endpoint: str, insecure: bool) -> None:
+    exporter = OTLPMetricExporter(endpoint=endpoint or None, insecure=insecure)
+    reader = PeriodicExportingMetricReader(exporter)
+    provider = MeterProvider(resource=_resource(service_name, environment), metric_readers=[reader])
+    metrics.set_meter_provider(provider)
+
+
+def instrument_flask_app(app) -> None:
+    FlaskInstrumentor().instrument_app(app)
+
+
+def setup(app, service_name: str, environment: Optional[str] = None) -> None:
+    environment = environment or os.getenv("APP_ENV", "dev")
+    endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
+    insecure = os.getenv("OTEL_EXPORTER_OTLP_INSECURE", "true").lower() == "true"
+
+    try:
+        init_tracing(service_name, environment, endpoint, insecure)
+        init_metrics(service_name, environment, endpoint, insecure)
+        instrument_flask_app(app)
+    except Exception:  # pragma: no cover - best effort initialization
+        _logger.exception("failed to initialize telemetry")

app/tests/test_main.py

@@ -1,5 +1,6 @@
 import pytest
-from main import app
+
+from app.main import app
 
 
 @pytest.fixture

scripts/smoke-tests.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+COMPOSE_FILE=${COMPOSE_FILE:-docker-compose.yml}
+
+echo "Starting smoke test environment using ${COMPOSE_FILE}"
+docker compose -f "${COMPOSE_FILE}" up --build -d
+
+cleanup() {
+  echo "Stopping smoke test environment"
+  docker compose -f "${COMPOSE_FILE}" down -v
+}
+
+trap cleanup EXIT
+
+echo "Waiting for services to become healthy"
+sleep 10
+
+echo "Running health checks"
+docker compose -f "${COMPOSE_FILE}" exec app curl -sf http://localhost:5000/health
+docker compose -f "${COMPOSE_FILE}" exec authz curl -sf http://localhost:8080/health || true
+docker compose -f "${COMPOSE_FILE}" exec inventory curl -sf http://localhost:8080/health
+
+echo "Smoke tests completed successfully"