Skip to content

Commit cc43469

Browse files
committed
feat: added e2e tests for api
1 parent 14ce35f commit cc43469

File tree

47 files changed

+10347
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+10347
-0
lines changed
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# E2E API tests (Newman/Postman) – runs on every PR.
2+
# Requires provider API keys to be set in the repository secrets (or environment).
3+
name: E2E API Tests
4+
5+
on:
6+
pull_request:
7+
branches: [main, master]
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: e2e-api-${{ github.event.pull_request.number || github.ref }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
e2e-api-tests:
16+
name: E2E API (Newman)
17+
runs-on: ubuntu-latest
18+
permissions:
19+
contents: read
20+
21+
steps:
22+
- name: Checkout repository
23+
uses: actions/checkout@v4
24+
25+
- name: Set up Go
26+
uses: actions/setup-go@v5
27+
with:
28+
go-version: "1.25.5"
29+
30+
- name: Set up Node.js
31+
uses: actions/setup-node@v4
32+
with:
33+
node-version: "20"
34+
35+
- name: Install Newman
36+
run: npm install -g newman
37+
38+
- name: Start Docker services (PostgreSQL)
39+
run: |
40+
docker compose -f .github/workflows/configs/docker-compose.yml up -d postgres
41+
echo "Waiting for Postgres..."
42+
timeout 60 bash -c 'until docker exec $(docker compose -f .github/workflows/configs/docker-compose.yml ps -q postgres) pg_isready -U bifrost -d bifrost; do sleep 2; done'
43+
44+
- name: Build UI
45+
run: make build-ui
46+
47+
- name: Build hello-world plugin
48+
run: |
49+
cd examples/plugins/hello-world && make build && cd ../../..
50+
51+
- name: Build bifrost-http binary
52+
run: |
53+
mkdir -p tmp
54+
cd transports/bifrost-http && go build -o ../../tmp/bifrost-http . && cd ../..
55+
56+
- name: Make E2E API scripts executable
57+
run: |
58+
chmod +x .github/workflows/scripts/run-e2e-api-tests.sh
59+
chmod +x .github/workflows/scripts/start-bifrost-and-run-e2e-api-tests.sh
60+
chmod +x tests/e2e/api/run-all-integration-tests.sh
61+
chmod +x tests/e2e/api/run-newman-tests.sh
62+
chmod +x tests/e2e/api/run-newman-api-tests.sh
63+
chmod +x tests/e2e/api/run-newman-openai-integration.sh
64+
chmod +x tests/e2e/api/run-newman-anthropic-integration.sh
65+
chmod +x tests/e2e/api/run-newman-bedrock-integration.sh
66+
chmod +x tests/e2e/api/run-newman-composite-integration.sh
67+
68+
- name: Run E2E API tests
69+
env:
70+
POSTGRES_HOST: localhost
71+
POSTGRES_PORT: "5432"
72+
POSTGRES_USER: bifrost
73+
POSTGRES_PASSWORD: bifrost_password
74+
POSTGRES_DB: bifrost
75+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
76+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
77+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
78+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
79+
AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }}
80+
AWS_ARN: ${{ secrets.AWS_ARN }}
81+
AWS_REGION: ${{ secrets.AWS_REGION }}
82+
BEDROCK_API_KEY: ${{ secrets.BEDROCK_API_KEY }}
83+
AZURE_ENDPOINT: ${{ secrets.AZURE_ENDPOINT }}
84+
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
85+
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
86+
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
87+
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
88+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
89+
PARASAIL_API_KEY: ${{ secrets.PARASAIL_API_KEY }}
90+
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
91+
ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }}
92+
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
93+
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
94+
VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}
95+
VERTEX_PROJECT_ID: ${{ secrets.VERTEX_PROJECT_ID }}
96+
GOOGLE_LOCATION: ${{ secrets.GOOGLE_LOCATION }}
97+
HUGGING_FACE_API_KEY: ${{ secrets.HUGGING_FACE_API_KEY }}
98+
run: |
99+
./.github/workflows/scripts/start-bifrost-and-run-e2e-api-tests.sh ./tmp/bifrost-http 8080
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Run E2E API tests (Newman/Postman) against a running Bifrost server.
5+
# Expects: Bifrost server already running at BIFROST_BASE_URL (default http://localhost:8080)
6+
# All provider API keys set in environment (OPENAI_API_KEY, ANTHROPIC_API_KEY, etc.)
7+
# Usage: ./run-e2e-api-tests.sh
8+
9+
if command -v readlink >/dev/null 2>&1 && readlink -f "$0" >/dev/null 2>&1; then
10+
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
11+
else
12+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
13+
fi
14+
15+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd -P)"
16+
E2E_API_DIR="$REPO_ROOT/tests/e2e/api"
17+
18+
export BIFROST_BASE_URL="${BIFROST_BASE_URL:-http://localhost:8080}"
19+
20+
echo "🧪 Running E2E API tests (Newman)"
21+
echo " BIFROST_BASE_URL=$BIFROST_BASE_URL"
22+
echo ""
23+
24+
if ! command -v newman &>/dev/null; then
25+
echo "❌ Newman is not installed. Install with: npm install -g newman" >&2
26+
exit 1
27+
fi
28+
29+
# Optional: setup plugin and MCP for API Management / integration tests
30+
if [ -f "$E2E_API_DIR/setup-plugin.sh" ]; then
31+
echo "📦 Setting up test plugin (optional)..."
32+
"$E2E_API_DIR/setup-plugin.sh" 2>/dev/null || echo " Plugin setup skipped"
33+
fi
34+
if [ -f "$E2E_API_DIR/setup-mcp.sh" ]; then
35+
echo "🔌 Setting up test MCP server (optional)..."
36+
"$E2E_API_DIR/setup-mcp.sh" 2>/dev/null || echo " MCP setup skipped"
37+
fi
38+
echo ""
39+
40+
cd "$E2E_API_DIR"
41+
42+
# 1. API Management & Health + Integration collections (OpenAI, Anthropic, Bedrock, Composite)
43+
echo "=========================================="
44+
echo "Running Integration test suites..."
45+
echo "=========================================="
46+
if ! ./run-all-integration-tests.sh; then
47+
echo "❌ Integration test suites failed"
48+
exit 1
49+
fi
50+
51+
# 2. V1 complete collection (all providers from provider_config)
52+
echo ""
53+
echo "=========================================="
54+
echo "Running V1 API test suite..."
55+
echo "=========================================="
56+
if ! ./run-newman-tests.sh; then
57+
echo "❌ V1 API test suite failed"
58+
exit 1
59+
fi
60+
61+
echo ""
62+
echo "✅ All E2E API tests passed"
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Start Bifrost with merged config (providers from env + Postgres) and run E2E API tests (Newman).
5+
# Requires: Docker with Postgres already running (docker compose up -d)
6+
# All provider API keys in environment (OPENAI_API_KEY, ANTHROPIC_API_KEY, etc.)
7+
# Usage: ./start-bifrost-and-run-e2e-api-tests.sh <bifrost-binary-path> [port]
8+
9+
if command -v readlink >/dev/null 2>&1 && readlink -f "$0" >/dev/null 2>&1; then
10+
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
11+
else
12+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
13+
fi
14+
15+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd -P)"
16+
17+
if [ "${1:-}" = "" ]; then
18+
echo "Usage: $0 <bifrost-binary-path> [port]" >&2
19+
exit 1
20+
fi
21+
22+
BIFROST_BINARY="$1"
23+
PORT="${2:-8080}"
24+
25+
POSTGRES_HOST="${POSTGRES_HOST:-localhost}"
26+
POSTGRES_PORT="${POSTGRES_PORT:-5432}"
27+
POSTGRES_USER="${POSTGRES_USER:-bifrost}"
28+
POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-bifrost_password}"
29+
POSTGRES_DB="${POSTGRES_DB:-bifrost}"
30+
POSTGRES_SSLMODE="${POSTGRES_SSLMODE:-disable}"
31+
32+
if [ ! -f "$BIFROST_BINARY" ] || [ ! -x "$BIFROST_BINARY" ]; then
33+
echo "❌ Bifrost binary not found or not executable: $BIFROST_BINARY" >&2
34+
exit 1
35+
fi
36+
37+
# Base config has providers with env.OPENAI_API_KEY etc.; we merge Postgres for stores
38+
BASE_CONFIG="$REPO_ROOT/tests/integrations/python/config.json"
39+
if [ ! -f "$BASE_CONFIG" ]; then
40+
echo "❌ Base config not found: $BASE_CONFIG" >&2
41+
exit 1
42+
fi
43+
44+
TEMP_DIR=$(mktemp -d)
45+
MERGED_CONFIG="$TEMP_DIR/config.json"
46+
SERVER_LOG="$TEMP_DIR/server.log"
47+
BIFROST_PID=""
48+
49+
cleanup() {
50+
local exit_code=$?
51+
if [ -n "${BIFROST_PID:-}" ] && kill -0 "$BIFROST_PID" 2>/dev/null; then
52+
kill "$BIFROST_PID" 2>/dev/null || true
53+
wait "$BIFROST_PID" 2>/dev/null || true
54+
fi
55+
rm -rf "$TEMP_DIR"
56+
exit $exit_code
57+
}
58+
trap cleanup EXIT
59+
60+
echo "📝 Merged config (providers + Postgres)..."
61+
if command -v jq >/dev/null 2>&1; then
62+
jq --arg host "$POSTGRES_HOST" --arg port "$POSTGRES_PORT" --arg user "$POSTGRES_USER" \
63+
--arg pass "$POSTGRES_PASSWORD" --arg db "$POSTGRES_DB" --arg ssl "$POSTGRES_SSLMODE" \
64+
'. + {
65+
"config_store": {"enabled": true, "type": "postgres", "config": {"host": $host, "port": $port, "user": $user, "password": $pass, "db_name": $db, "ssl_mode": $ssl}},
66+
"logs_store": {"enabled": true, "type": "postgres", "config": {"host": $host, "port": $port, "user": $user, "password": $pass, "db_name": $db, "ssl_mode": $ssl}}
67+
}' "$BASE_CONFIG" > "$MERGED_CONFIG"
68+
else
69+
python3 - "$BASE_CONFIG" "$MERGED_CONFIG" << EOF
70+
import sys, json, os
71+
with open(sys.argv[1]) as f: c = json.load(f)
72+
pg = {"host": os.environ.get("POSTGRES_HOST", "localhost"), "port": os.environ.get("POSTGRES_PORT", "5432"), "user": os.environ.get("POSTGRES_USER", "bifrost"), "password": os.environ.get("POSTGRES_PASSWORD", "bifrost_password"), "db_name": os.environ.get("POSTGRES_DB", "bifrost"), "ssl_mode": os.environ.get("POSTGRES_SSLMODE", "disable")}
73+
c["config_store"] = {"enabled": True, "type": "postgres", "config": pg}
74+
c["logs_store"] = {"enabled": True, "type": "postgres", "config": dict(pg)}
75+
with open(sys.argv[2], "w") as f: json.dump(c, f, indent=2)
76+
EOF
77+
fi
78+
79+
echo "🔄 Resetting PostgreSQL database..."
80+
DOCKER_COMPOSE_FILE="$REPO_ROOT/.github/workflows/configs/docker-compose.yml"
81+
if [ -f "$DOCKER_COMPOSE_FILE" ]; then
82+
POSTGRES_CONTAINER=$(docker compose -f "$DOCKER_COMPOSE_FILE" ps -q postgres 2>/dev/null || true)
83+
if [ -n "$POSTGRES_CONTAINER" ]; then
84+
docker exec "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" -d postgres -c "DROP DATABASE IF EXISTS $POSTGRES_DB; CREATE DATABASE $POSTGRES_DB;" 2>/dev/null || true
85+
fi
86+
fi
87+
88+
echo "🚀 Starting Bifrost on port $PORT..."
89+
"$BIFROST_BINARY" --app-dir "$TEMP_DIR" --port "$PORT" --log-level debug > "$SERVER_LOG" 2>&1 &
90+
BIFROST_PID=$!
91+
92+
MAX_WAIT=60
93+
ELAPSED=0
94+
while [ $ELAPSED -lt $MAX_WAIT ]; do
95+
if grep -q "successfully started bifrost" "$SERVER_LOG" 2>/dev/null; then
96+
echo " ✅ Bifrost started"
97+
break
98+
fi
99+
if ! kill -0 "$BIFROST_PID" 2>/dev/null; then
100+
echo " ❌ Bifrost process exited"
101+
cat "$SERVER_LOG"
102+
exit 1
103+
fi
104+
sleep 1
105+
ELAPSED=$((ELAPSED + 1))
106+
done
107+
108+
if [ $ELAPSED -ge $MAX_WAIT ]; then
109+
echo " ❌ Bifrost did not start within ${MAX_WAIT}s"
110+
cat "$SERVER_LOG"
111+
exit 1
112+
fi
113+
114+
export BIFROST_BASE_URL="http://localhost:$PORT"
115+
"$SCRIPT_DIR/run-e2e-api-tests.sh"

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ transports/schema/config.schema.json
4747

4848
# Test reports
4949
test-reports
50+
tests/e2e/api/newman-reports
5051

5152
# AI Agent specific
5253
.claude/

0 commit comments

Comments
 (0)