add simple agent integration test

.github/workflows/integration-testing.yml

@@ -0,0 +1,73 @@
+name: Secure Integration test
+
+on:
+  pull_request:
+  pull_request_target:
+    branches: main
+
+jobs:
+  authorization-check:
+    permissions: read-all
+    runs-on: ubuntu-latest
+    outputs:
+      approval-env: ${{ steps.collab-check.outputs.result }}
+    steps:
+      - name: Collaborator Check
+        uses: actions/github-script@v7
+        id: collab-check
+        with:
+          result-encoding: string
+          script: |
+            try {
+              const permissionResponse = await github.rest.repos.getCollaboratorPermissionLevel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                username: context.payload.pull_request.user.login,
+              });
+              const permission = permissionResponse.data.permission;
+              const hasWriteAccess = ['write', 'admin'].includes(permission);
+              if (!hasWriteAccess) {
+                console.log(`User ${context.payload.pull_request.user.login} does not have write access to the repository (permission: ${permission})`);
+                return "manual-approval"
+              } else {
+                console.log(`Verifed ${context.payload.pull_request.user.login} has write access. Auto Approving PR Checks.`)
+                return "auto-approve"
+              }
+            } catch (error) {
+              console.log(`${context.payload.pull_request.user.login} does not have write access. Requiring Manual Approval to run PR Checks.`)
+              return "manual-approval"
+            }
+  check-access-and-checkout:
+    runs-on: ubuntu-latest
+    needs: authorization-check
+    environment: ${{ needs.authorization-check.outputs.approval-env }}
+    permissions:
+      id-token: write
+      pull-requests: read
+      contents: read
+    steps:
+      - name: Configure Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+         role-to-assume: ${{ secrets.AGENTCORE_INTEG_TEST_ROLE }}
+         aws-region: us-west-2
+         mask-aws-account-id: true
+      - name: Checkout head commit
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }} # Pull the commit from the forked repo
+          persist-credentials: false  # Don't persist credentials for subsequent actions
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          pip install -e .
+          pip install --no-cache-dir pytest requests strands-agents
+      - name: Run integration tests
+        env:
+          AWS_REGION: us-west-2
+        id: tests
+        run: |
+          pytest tests_integ/runtime -s --log-cli-level=INFO

pyproject.toml

@@ -142,4 +142,5 @@ dev = [
     "pytest-cov>=6.0.0",
     "ruff>=0.12.0",
     "wheel>=0.45.1",
+    "strands-agents>=1.1.0",
 ]

tests_integ/runtime/base_test.py

@@ -0,0 +1,103 @@
+import logging
+import os
+import subprocess
+import threading
+import time
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from subprocess import Popen
+from typing import IO, Generator
+
+logger = logging.getLogger("sdk-runtime-base-test")
+
+AGENT_SERVER_ENDPOINT = "http://127.0.0.1:8080"
+
+
+class BaseSDKRuntimeTest(ABC):
+    def run(self, tmp_path) -> None:
+        original_dir = os.getcwd()
+        try:
+            os.chdir(tmp_path)
+
+            self.setup()
+
+            logger.info("Running test...")
+            self.run_test()
+
+        finally:
+            os.chdir(original_dir)
+
+    def setup(self) -> None:
+        return
+
+    @abstractmethod
+    def run_test(self) -> None:
+        raise NotImplementedError
+
+
+@contextmanager
+def start_agent_server(agent_module, timeout=5) -> Generator[Popen, None, None]:
+    logger.info("Starting agent server...")
+    start_time = time.time()
+
+    try:
+        agent_server = Popen(
+            ["python", "-m", agent_module], text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+        )
+
+        while time.time() - start_time < timeout:
+            if agent_server.stdout is None:
+                raise RuntimeError("Agent server has no configured output")
+
+            if agent_server.poll() is not None:
+                out = agent_server.stdout.read()
+                raise RuntimeError(f"Error when running agent server: {out}")
+
+            line = agent_server.stdout.readline()
+            while line:
+                line = line.strip()
+                if line:
+                    logger.info(line)
+                    if "Uvicorn running on http://127.0.0.1:8080" in line:
+                        _start_logging_thread(agent_server.stdout)
+                        yield agent_server
+                        return
+                line = agent_server.stdout.readline()
+
+            time.sleep(0.5)
+        raise TimeoutError(f"Agent server did not start within {timeout} seconds")
+    finally:
+        _stop_agent_server(agent_server)
+
+
+def _stop_agent_server(agent_server: Popen) -> None:
+    logger.info("Stopping agent server...")
+    if agent_server.poll() is None:  # Process is still running
+        logger.info("Terminating agent server process...")
+        agent_server.terminate()
+
+        # Wait for graceful shutdown
+        try:
+            agent_server.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            logger.warning("Agent server didn't terminate, force killing...")
+            agent_server.kill()
+            agent_server.wait()
+        finally:
+            if agent_server.stdout:
+                agent_server.stdout.close()
+    logger.info("Agent server terminated")
+
+
+def _start_logging_thread(stdout: IO[str]):
+    def log_server_output():
+        logger.info("Server logging thread started")
+        # thread is stopped when stdout is closed
+        for line in iter(stdout.readline, ""):
+            if line.strip():
+                logger.info(line.strip())
+        logger.info("Server logging thread stopped")
+
+    logging_thread = threading.Thread(target=log_server_output, daemon=True, name="AgentServerLogger")
+    logging_thread.start()
+    return logging_thread

tests_integ/runtime/http_client.py

@@ -0,0 +1,47 @@
+import json
+import logging
+
+import requests
+
+
+class HttpClient:
+    """Local HTTP client for invoking endpoints."""
+
+    def __init__(self, endpoint: str):
+        """Initialize the local client with the given endpoint."""
+        self.endpoint = endpoint
+        self.logger = logging.getLogger("sdk-runtime-test-http-client")
+
+    def invoke_endpoint(self, payload: str):
+        """Invoke the endpoint with the given parameters."""
+        self.logger.info("Sending request to agent with payload: %s", payload)
+
+        url = f"{self.endpoint}/invocations"
+
+        headers = {
+            "Content-Type": "application/json",
+        }
+
+        try:
+            body = json.loads(payload) if isinstance(payload, str) else payload
+        except json.JSONDecodeError:
+            # Fallback for non-JSON strings - wrap in payload object
+            self.logger.warning("Failed to parse payload as JSON, wrapping in payload object")
+            body = {"message": payload}
+
+        try:
+            # Make request with timeout
+            return requests.post(url, headers=headers, json=body, timeout=100, stream=True).text
+        except requests.exceptions.RequestException as e:
+            self.logger.error("Failed to invoke agent endpoint: %s", str(e))
+            raise
+
+    def ping(self):
+        self.logger.info("Pinging agent server")
+
+        url = f"{self.endpoint}/ping"
+        try:
+            return requests.get(url, timeout=2).text
+        except requests.exceptions.RequestException as e:
+            self.logger.error("Failed to ping agent endpoint: %s", str(e))
+            raise

tests_integ/runtime/test_simple_agent.py

@@ -0,0 +1,43 @@
+import logging
+import textwrap
+
+from tests_integ.runtime.base_test import AGENT_SERVER_ENDPOINT, BaseSDKRuntimeTest, start_agent_server
+from tests_integ.runtime.http_client import HttpClient
+
+logger = logging.getLogger("sdk-runtime-simple-agent-test")
+
+
+class TestSDKSimpleAgent(BaseSDKRuntimeTest):
+    def setup(self):
+        self.agent_module = "agent"
+        with open(self.agent_module + ".py", "w") as file:
+            content = textwrap.dedent("""
+                from bedrock_agentcore import BedrockAgentCoreApp
+                from strands import Agent
+
+                app = BedrockAgentCoreApp()
+                agent = Agent()
+
+                @app.entrypoint
+                async def agent_invocation(payload):
+                    return agent(payload.get("message"))
+
+                app.run()
+            """).strip()
+            file.write(content)
+
+    def run_test(self):
+        with start_agent_server(self.agent_module):
+            client = HttpClient(AGENT_SERVER_ENDPOINT)
+
+            ping_response = client.ping()
+            logger.info(ping_response)
+            assert "Healthy" in ping_response
+
+            response = client.invoke_endpoint("tell me a joke")
+            logger.info(response)
+            assert "Because they make up everything!" in response
+
+
+def test(tmp_path):
+    TestSDKSimpleAgent().run(tmp_path)