From f0950d5df840a760b98f14d76c0b7f2be17ef9c4 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 10:59:20 -0700 Subject: [PATCH 01/70] added new hocon --- .../music_nerd_pro_llm_bedrock-claude.hocon | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon diff --git a/neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon b/neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon new file mode 100644 index 000000000..0770789c7 --- /dev/null +++ b/neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon @@ -0,0 +1,96 @@ + +# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. +# All Rights Reserved. +# Issued under the Academic Public License. +# +# You can be released from the terms, and requirements of the Academic Public +# License by purchasing a commercial license. +# Purchase of a commercial license is mandatory for any use of the +# neuro-san SDK Software in commercial settings. +# +# END COPYRIGHT + +# The schema specifications for this file are documented here: +# https://github.com/cognizant-ai-lab/neuro-san/blob/main/docs/agent_hocon_reference.md + +{ + "llm_config": { + "model_name": "bedrock-us-claude-3-7-sonnet", + "credentials_profile_name": "daiDev", + "region_name": "us-east-1" + }, + "tools": [ + # These tool definitions do not have to be in any particular order + # How they are linked and call each other is defined within their + # own specs. This could be a graph, potentially even with cycles. + + # This first agent definition is regarded as the "Front Man", which + # does all the talking to the outside world/client. + # It is identified as such because it is either: + # A) The only one with no parameters in his function definition, + # and therefore he needs to talk to the outside world to get things rolling. + # B) The first agent listed, regardless of function parameters. + # + # Some disqualifications from being a front man: + # 1) Cannot use a CodedTool "class" definition + # 2) Cannot use a Tool "toolbox" definition + { + "name": "MusicNerdPro", + + # Note that there are no parameters defined for this guy's "function" key. + # This is the primary way to identify this tool as a front-man, + # distinguishing it from the rest of the tools. + + "function": { + + # The description acts as an initial prompt. + "description": """ +I can help with music-related inquiries. +""" + }, + + "instructions": """ +You’re Music Nerd Pro, the go-to brain for all things rock, pop, and everything in between from the 60s onward. You live for liner notes, B-sides, lost demos, and legendary live sets. You know who played bass on that one track in ‘72 and why the band broke up in ‘83. People come to you with questions like: + • “What’s the story behind this song?” + • “Which album should I start with?” + • “Who influenced this band’s sound?” + • “Is there a deeper meaning in these lyrics?” + • “What’s a hidden gem I probably missed?” +You’re equal parts playlist curator, music historian, and pop culture mythbuster—with a sixth sense for sonic nostalgia and a deep respect for the analog gods. + +- You must call the Accountant tool exactly once per user question — no more, no less. +- You must not estimate, guess, or invent the cost under any circumstance. +- You must not skip calling the Accountant tool — it is required once for every question. + +For each question you receive, call your Accountant agent to calculate the running cost. Otherwise you won't get paid! +Then answer with a JSON message that has two keys: +1. An "answer" key whose value has the answer to the question +2. A "running_cost" key whose value has the running cost computed by the Accountant agent. +""", + "tools": ["Accountant"] + + # Parse any JSON in responses into the structure field of the ChatMessage. + "structure_formats": "json" + }, + { + "name": "Accountant", + "function": { + "description": """ +You are an API that keeps track of the running cost of the MusicNerdPro service. Pass the current running cost +to the API to get the updated running cost. If no running cost it known, pass 0.00. + """, + "parameters": { + "type": "object", + "properties": { + "running_cost": { + "type": "float", + "description": "The current running total of the service cost." + }, + }, + "required": ["running_cost"] + } + }, + "class": "accountant.Accountant" + }, + ] +} From 5d041dfae03f8367aa244b03ccd6063d23d90cec Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 11:01:17 -0700 Subject: [PATCH 02/70] regrister new hocon --- neuro_san/registries/manifest.hocon | 1 + 1 file changed, 1 insertion(+) diff --git a/neuro_san/registries/manifest.hocon b/neuro_san/registries/manifest.hocon index ee8f79131..b4fa9f9f8 100644 --- a/neuro_san/registries/manifest.hocon +++ b/neuro_san/registries/manifest.hocon @@ -29,6 +29,7 @@ "music_nerd_pro_llm_azure.hocon": true, "music_nerd_pro_llm_ollama.hocon": true, "music_nerd_pro_multi_agents.hocon": true, + "music_nerd_pro_llm_bedrock-claude.hocon": true, # This one is an example of agents calling tools that has no parameters. "date_time.hocon": true, From c3d20f4c0d2ed583d8add0f7b91b1bd29ef81a98 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 11:04:23 -0700 Subject: [PATCH 03/70] Added test fixtures for new hoc test case --- ...nation_responses_with_history_direct.hocon | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/fixtures/music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon diff --git a/tests/fixtures/music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon b/tests/fixtures/music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon new file mode 100644 index 000000000..7cd472004 --- /dev/null +++ b/tests/fixtures/music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon @@ -0,0 +1,62 @@ +# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. +# All Rights Reserved. +# Issued under the Academic Public License. +# +# You can be released from the terms, and requirements of the Academic Public +# License by purchasing a commercial license. +# Purchase of a commercial license is mandatory for any use of the +# neuro-san SDK Software in commercial settings. +# +# END COPYRIGHT + +# This file defines everything necessary for a data-driven test. +{ + # Describes what agent to test against. + "agent": "music_nerd_pro_llm_bedrock-claude", + + # Connect to the agent via a server + "connections": ["direct"], + + # Interactions are a series of dictionaries with request elements paired with + # descriptions of response checks. + "interactions": [ + { + # This is what we send as input to streaming_chat() + "text": "Who did Yellow Submarine?", + + # The response block treats how we are going to test what comes back + "response": { + # Structure block says how we are going to examine the structure + # (dictionary) returned as part of the response. + "structure": { + # "answer" is a key that is supposed to be in the dictionary. + "answer": { + # Keywords says we are going to look for exact matches for each + # element in a list of strings. All elements need to show up + # in order to pass. + "keywords": "Beatles" + }, + "running_cost": { + "value": 3.0 + } + } + } + }, + { + # This next interaction block tests the ability for an agent + # to continue its conversation with the context from the previous + # interaction. + "text": "Where were they from?", + "response": { + "structure": { + "answer": { + "keywords": "Liverpool" + }, + "running_cost": { + "value": 6.0 + } + } + } + } + ] +} From fc6225acb7774f8cc9c962dedd63dcba228409a8 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 11:06:37 -0700 Subject: [PATCH 04/70] Added coded_tools for new hocon --- .../__init__.py | 0 .../accountant.py | 48 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/__init__.py create mode 100644 neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/accountant.py diff --git a/neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/__init__.py b/neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/accountant.py b/neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/accountant.py new file mode 100644 index 000000000..f7b4bbbc0 --- /dev/null +++ b/neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/accountant.py @@ -0,0 +1,48 @@ +from typing import Any +from typing import Dict +from typing import Union + +from neuro_san.interfaces.coded_tool import CodedTool + + +class Accountant(CodedTool): + """ + A tool that updates a running cost each time it is called. + """ + + def invoke(self, args: Dict[str, Any], sly_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Updates the passed running cost each time it's called. + :param args: A dictionary with the following keys: + "running_cost": the running cost to update. + + :param sly_data: A dictionary containing parameters that should be kept out of the chat stream. + Keys expected for this implementation are: + None + + :return: A dictionary containing: + "running_cost": the updated running cost. + """ + tool_name = self.__class__.__name__ + print(f"========== Calling {tool_name} ==========") + # Parse the arguments + print(f"args: {args}") + running_cost: float = float(args.get("running_cost")) + + # Increment the running cost not using value other 1 + # This would make a little hard if the LLM wanted to guess + updated_running_cost: float = running_cost + 3.0 + + tool_response = { + "running_cost": updated_running_cost + } + print("-----------------------") + print(f"{tool_name} response: ", tool_response) + print(f"========== Done with {tool_name} ==========") + return tool_response + + async def async_invoke(self, args: Dict[str, Any], sly_data: Dict[str, Any]) -> Union[Dict[str, Any], str]: + """ + Delegates to the synchronous invoke method because it's quick, non-blocking. + """ + return self.invoke(args, sly_data) From 272f10d3d9a92ec0877e49d58c9324f22f4f97ed Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 11:35:30 -0700 Subject: [PATCH 05/70] added new test case to smoke test list --- tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py index 604fa65d5..f97fcdcc1 100644 --- a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py +++ b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py @@ -87,6 +87,7 @@ def test_hocon_with_server(self, test_name: str, test_hocon: str): "music_nerd_pro_llm_anthropic/combination_responses_with_history_direct.hocon", "music_nerd_pro_llm_gemini/combination_responses_with_history_direct.hocon", "music_nerd_pro_llm_azure/combination_responses_with_history_direct.hocon", + "music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon", # List more hocon files as they become available here. ])) From e39508a45598f2caa1f9982a08a0898e5b5bc991 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 17:03:18 -0700 Subject: [PATCH 06/70] added Bedrock-claude LLM smoke test case --- neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon b/neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon index 0770789c7..bad37b0a5 100644 --- a/neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon +++ b/neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon @@ -17,7 +17,7 @@ "llm_config": { "model_name": "bedrock-us-claude-3-7-sonnet", "credentials_profile_name": "daiDev", - "region_name": "us-east-1" + "region_name": "us-west-2" }, "tools": [ # These tool definitions do not have to be in any particular order From 2b8860c9796db222d08b4577e5e902db3c28f928 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 17:04:37 -0700 Subject: [PATCH 07/70] Removed class from gemini hocon --- neuro_san/registries/music_nerd_pro_llm_gemini.hocon | 1 - 1 file changed, 1 deletion(-) diff --git a/neuro_san/registries/music_nerd_pro_llm_gemini.hocon b/neuro_san/registries/music_nerd_pro_llm_gemini.hocon index c4954b497..2b6d23bd5 100644 --- a/neuro_san/registries/music_nerd_pro_llm_gemini.hocon +++ b/neuro_san/registries/music_nerd_pro_llm_gemini.hocon @@ -15,7 +15,6 @@ { "llm_config": { - "class": "gemini", "model_name": "gemini-2.5-flash", }, "tools": [ From e3a9d027bc8f6faf005970eb06aae283d5c853a4 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 17:36:54 -0700 Subject: [PATCH 08/70] Renamed new test case --- .../__init__.py | 0 .../accountant.py | 0 neuro_san/registries/manifest.hocon | 2 +- ...ck-claude.hocon => music_nerd_pro_llm_bedrock_claude.hocon} | 0 .../combination_responses_with_history_direct.hocon | 2 +- tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py | 3 ++- 6 files changed, 4 insertions(+), 3 deletions(-) rename neuro_san/coded_tools/{music_nerd_pro_llm_bedrock-claude => music_nerd_pro_llm_bedrock_claude}/__init__.py (100%) rename neuro_san/coded_tools/{music_nerd_pro_llm_bedrock-claude => music_nerd_pro_llm_bedrock_claude}/accountant.py (100%) rename neuro_san/registries/{music_nerd_pro_llm_bedrock-claude.hocon => music_nerd_pro_llm_bedrock_claude.hocon} (100%) rename tests/fixtures/{music_nerd_pro_llm_bedrock-claude => music_nerd_pro_llm_bedrock_claude}/combination_responses_with_history_direct.hocon (97%) diff --git a/neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/__init__.py b/neuro_san/coded_tools/music_nerd_pro_llm_bedrock_claude/__init__.py similarity index 100% rename from neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/__init__.py rename to neuro_san/coded_tools/music_nerd_pro_llm_bedrock_claude/__init__.py diff --git a/neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/accountant.py b/neuro_san/coded_tools/music_nerd_pro_llm_bedrock_claude/accountant.py similarity index 100% rename from neuro_san/coded_tools/music_nerd_pro_llm_bedrock-claude/accountant.py rename to neuro_san/coded_tools/music_nerd_pro_llm_bedrock_claude/accountant.py diff --git a/neuro_san/registries/manifest.hocon b/neuro_san/registries/manifest.hocon index b4fa9f9f8..b36ffcb23 100644 --- a/neuro_san/registries/manifest.hocon +++ b/neuro_san/registries/manifest.hocon @@ -29,7 +29,7 @@ "music_nerd_pro_llm_azure.hocon": true, "music_nerd_pro_llm_ollama.hocon": true, "music_nerd_pro_multi_agents.hocon": true, - "music_nerd_pro_llm_bedrock-claude.hocon": true, + "music_nerd_pro_llm_bedrock_claude.hocon": true, # This one is an example of agents calling tools that has no parameters. "date_time.hocon": true, diff --git a/neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon b/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon similarity index 100% rename from neuro_san/registries/music_nerd_pro_llm_bedrock-claude.hocon rename to neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon diff --git a/tests/fixtures/music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon b/tests/fixtures/music_nerd_pro_llm_bedrock_claude/combination_responses_with_history_direct.hocon similarity index 97% rename from tests/fixtures/music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon rename to tests/fixtures/music_nerd_pro_llm_bedrock_claude/combination_responses_with_history_direct.hocon index 7cd472004..3b6a33b0f 100644 --- a/tests/fixtures/music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon +++ b/tests/fixtures/music_nerd_pro_llm_bedrock_claude/combination_responses_with_history_direct.hocon @@ -12,7 +12,7 @@ # This file defines everything necessary for a data-driven test. { # Describes what agent to test against. - "agent": "music_nerd_pro_llm_bedrock-claude", + "agent": "music_nerd_pro_llm_bedrock_claude", # Connect to the agent via a server "connections": ["direct"], diff --git a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py index f97fcdcc1..516f527d4 100644 --- a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py +++ b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py @@ -87,7 +87,7 @@ def test_hocon_with_server(self, test_name: str, test_hocon: str): "music_nerd_pro_llm_anthropic/combination_responses_with_history_direct.hocon", "music_nerd_pro_llm_gemini/combination_responses_with_history_direct.hocon", "music_nerd_pro_llm_azure/combination_responses_with_history_direct.hocon", - "music_nerd_pro_llm_bedrock-claude/combination_responses_with_history_direct.hocon", + "music_nerd_pro_llm_bedrock_claude/combination_responses_with_history_direct.hocon", # List more hocon files as they become available here. ])) @@ -97,6 +97,7 @@ def test_hocon_with_server(self, test_name: str, test_hocon: str): @pytest.mark.anthropic @pytest.mark.gemini @pytest.mark.azure + @pytest.mark.bedrock_claude def test_hocon_with_non_default_llm(self, test_name: str, test_hocon: str): """ Test method for a single parameterized test case specified by a hocon file. From 70eaf3cd92f614a02aa54ea551a8cf94d6b007f8 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Fri, 25 Jul 2025 18:07:47 -0700 Subject: [PATCH 09/70] tweak warning --- pytest.ini | 10 ++++++---- tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pytest.ini b/pytest.ini index c27fea953..f98de6c4f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,10 +7,12 @@ markers = non_default_llm_provider: Tests that exercise llm providers other than our default Open AI anthropic: Tests that specifically use anthropic as the llm provider gemini: Tests that specifically use gemini as the llm provider - azure: Tests that specifically use gemini as the llm provider - ollama: Tests that specifically use gemini as the llm provider + azure: Tests that specifically use azure as the llm provider + ollama: Tests that specifically use ollama as the llm provider + bedrock_claude: mark tests that target the Bedrock‑Claude LLM backend needs_server: Tests that need a server running in order to complete successfully +# silence specific warnings filterwarnings = - # Ignore warnings about protobuf 4 - ignore:Type google._upb._message.* uses PyType_Spec with a metaclass that has custom tp_new:DeprecationWarning + # Ignore warnings about protobuf 4's PyType_Spec metaclass usage + ignore:Type google\._upb\._message.* uses PyType_Spec with a metaclass that has custom tp_new:DeprecationWarning diff --git a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py index 516f527d4..561091e69 100644 --- a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py +++ b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py @@ -91,7 +91,7 @@ def test_hocon_with_server(self, test_name: str, test_hocon: str): # List more hocon files as they become available here. ])) - @pytest.mark.timeout(30) # in seconds for this test + @pytest.mark.timeout(60) # in seconds for this test @pytest.mark.smoke @pytest.mark.non_default_llm_provider @pytest.mark.anthropic From 66f778797a63190d5d096a8cbd679d6d8082652d Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Mon, 28 Jul 2025 16:00:41 -0700 Subject: [PATCH 10/70] Added Set up AWS config & credentials --- .github/workflows/smoke.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 33daf0aab..b5a6ab759 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -60,9 +60,34 @@ jobs: AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AGENT_TOOL_PATH: "./neuro_san/coded_tools" PYTHONPATH: ${{ env.PYTHONPATH }}:. run: | + # Set up AWS config & credentials + mkdir -p ~/.aws + + # AWS CLI config with default and daiDev profile + cat > ~/.aws/config < ~/.aws/credentials < Date: Mon, 28 Jul 2025 16:11:30 -0700 Subject: [PATCH 11/70] replaced the expected vs. debug --- .github/workflows/smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index b5a6ab759..7fa717c09 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -75,7 +75,7 @@ jobs: output=json [profile daiDev] - role_arn=arn:aws:iam::111111:role/testonly + role_arn=arn:aws:iam::613076437200:role/LeafDev region=us-west-2 source_profile=default EOF From e8588054ebb782e3f8537b9360bbe18bb1a11d0c Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Mon, 28 Jul 2025 16:41:11 -0700 Subject: [PATCH 12/70] debug GitHub test case failure --- .github/workflows/smoke.yml | 24 +-- .../zzz_hocons/test_smoke_test_hocons.py | 158 +++++++++--------- 2 files changed, 91 insertions(+), 91 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 7fa717c09..6c5c28694 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -27,15 +27,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Check for NVIDIA GPU - shell: bash - run: | - build_scripts/check_gpu.sh + # - name: Check for NVIDIA GPU + # shell: bash + # run: | + # build_scripts/check_gpu.sh - - name: check available memory - shell: bash - run: | - build_scripts/check_memory.sh + # - name: check available memory + # shell: bash + # run: | + # build_scripts/check_memory.sh - name: Install dependencies run: | @@ -43,10 +43,10 @@ jobs: pip install -r requirements-build.txt pip install -r requirements.txt - - name: Set Ollama model to be used for related test case - shell: bash - run: | - build_scripts/set_ollama_model.sh + # - name: Set Ollama model to be used for related test case + # shell: bash + # run: | + # build_scripts/set_ollama_model.sh - name: Show installed packages run: pip freeze diff --git a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py index 561091e69..536af7a60 100644 --- a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py +++ b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py @@ -29,64 +29,64 @@ class TestSmokeTestHocons(TestCase): # annotation below so the instance can find the hocon test cases listed. DYNAMIC = DynamicHoconUnitTests(__file__, path_to_basis="../../fixtures") - @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ - # These can be in any order. - # Ideally more basic functionality will come first. - # Barring that, try to stick to alphabetical order. - "music_nerd_pro/combination_responses_with_history_direct.hocon", - - # List more hocon files as they become available here. - ])) - @pytest.mark.timeout(30) # 30 seconds for this test - @pytest.mark.smoke - def test_hocon(self, test_name: str, test_hocon: str): - """ - Test method for a single parameterized test case specified by a hocon file. - Arguments to this method are given by the iteration that happens as a result - of the magic of the @parameterized.expand annotation above. - - :param test_name: The name of a single test. - :param test_hocon: The hocon file of a single data-driven test case. - """ - # Call the guts of the dynamic test driver. - # This will expand the test_hocon file name from the expanded list to - # include the file basis implied by the __file__ and path_to_basis above. - self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) - - @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ - # These can be in any order. - # Ideally more basic functionality will come first. - # Barring that, try to stick to alphabetical order. - # http has been covered by integration test - # "music_nerd_pro/combination_responses_with_history_http.hocon", - "music_nerd_pro/combination_responses_with_history_grpc.hocon", - - # List more hocon files as they become available here. - ])) - @pytest.mark.timeout(30) # 30 seconds for this test - @pytest.mark.smoke - @pytest.mark.needs_server - def test_hocon_with_server(self, test_name: str, test_hocon: str): - """ - Test method for a single parameterized test case specified by a hocon file. - Arguments to this method are given by the iteration that happens as a result - of the magic of the @parameterized.expand annotation above. - - :param test_name: The name of a single test. - :param test_hocon: The hocon file of a single data-driven test case. - """ - # Call the guts of the dynamic test driver. - # This will expand the test_hocon file name from the expanded list to - # include the file basis implied by the __file__ and path_to_basis above. - self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) + # @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ + # # These can be in any order. + # # Ideally more basic functionality will come first. + # # Barring that, try to stick to alphabetical order. + # "music_nerd_pro/combination_responses_with_history_direct.hocon", + + # # List more hocon files as they become available here. + # ])) + # @pytest.mark.timeout(30) # 30 seconds for this test + # @pytest.mark.smoke + # def test_hocon(self, test_name: str, test_hocon: str): + # """ + # Test method for a single parameterized test case specified by a hocon file. + # Arguments to this method are given by the iteration that happens as a result + # of the magic of the @parameterized.expand annotation above. + + # :param test_name: The name of a single test. + # :param test_hocon: The hocon file of a single data-driven test case. + # """ + # # Call the guts of the dynamic test driver. + # # This will expand the test_hocon file name from the expanded list to + # # include the file basis implied by the __file__ and path_to_basis above. + # self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) + + # @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ + # # These can be in any order. + # # Ideally more basic functionality will come first. + # # Barring that, try to stick to alphabetical order. + # # http has been covered by integration test + # # "music_nerd_pro/combination_responses_with_history_http.hocon", + # "music_nerd_pro/combination_responses_with_history_grpc.hocon", + + # # List more hocon files as they become available here. + # ])) + # @pytest.mark.timeout(30) # 30 seconds for this test + # @pytest.mark.smoke + # @pytest.mark.needs_server + # def test_hocon_with_server(self, test_name: str, test_hocon: str): + # """ + # Test method for a single parameterized test case specified by a hocon file. + # Arguments to this method are given by the iteration that happens as a result + # of the magic of the @parameterized.expand annotation above. + + # :param test_name: The name of a single test. + # :param test_hocon: The hocon file of a single data-driven test case. + # """ + # # Call the guts of the dynamic test driver. + # # This will expand the test_hocon file name from the expanded list to + # # include the file basis implied by the __file__ and path_to_basis above. + # self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ # These can be in any order. # Ideally more basic functionality will come first. # Barring that, try to stick to alphabetical order. - "music_nerd_pro_llm_anthropic/combination_responses_with_history_direct.hocon", - "music_nerd_pro_llm_gemini/combination_responses_with_history_direct.hocon", - "music_nerd_pro_llm_azure/combination_responses_with_history_direct.hocon", + # "music_nerd_pro_llm_anthropic/combination_responses_with_history_direct.hocon", + # "music_nerd_pro_llm_gemini/combination_responses_with_history_direct.hocon", + # "music_nerd_pro_llm_azure/combination_responses_with_history_direct.hocon", "music_nerd_pro_llm_bedrock_claude/combination_responses_with_history_direct.hocon", # List more hocon files as they become available here. @@ -112,29 +112,29 @@ def test_hocon_with_non_default_llm(self, test_name: str, test_hocon: str): # include the file basis implied by the __file__ and path_to_basis above. self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) - @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ - # These can be in any order. - # Ideally more basic functionality will come first. - # Barring that, try to stick to alphabetical order. - "music_nerd_pro_llm_ollama/combination_responses_with_history_http.hocon", - - # List more hocon files as they become available here. - ])) - @pytest.mark.timeout(12 * 60) # in mins for this test - @pytest.mark.smoke - @pytest.mark.needs_server - @pytest.mark.non_default_llm_provider - @pytest.mark.ollama - def test_hocon_with_server_non_default_llm(self, test_name: str, test_hocon: str): - """ - Test method for a single parameterized test case specified by a hocon file. - Arguments to this method are given by the iteration that happens as a result - of the magic of the @parameterized.expand annotation above. - - :param test_name: The name of a single test. - :param test_hocon: The hocon file of a single data-driven test case. - """ - # Call the guts of the dynamic test driver. - # This will expand the test_hocon file name from the expanded list to - # include the file basis implied by the __file__ and path_to_basis above. - self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) + # @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ + # # These can be in any order. + # # Ideally more basic functionality will come first. + # # Barring that, try to stick to alphabetical order. + # "music_nerd_pro_llm_ollama/combination_responses_with_history_http.hocon", + + # # List more hocon files as they become available here. + # ])) + # @pytest.mark.timeout(12 * 60) # in mins for this test + # @pytest.mark.smoke + # @pytest.mark.needs_server + # @pytest.mark.non_default_llm_provider + # @pytest.mark.ollama + # def test_hocon_with_server_non_default_llm(self, test_name: str, test_hocon: str): + # """ + # Test method for a single parameterized test case specified by a hocon file. + # Arguments to this method are given by the iteration that happens as a result + # of the magic of the @parameterized.expand annotation above. + + # :param test_name: The name of a single test. + # :param test_hocon: The hocon file of a single data-driven test case. + # """ + # # Call the guts of the dynamic test driver. + # # This will expand the test_hocon file name from the expanded list to + # # include the file basis implied by the __file__ and path_to_basis above. + # self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) From 6da8c36a3f7feb7253103367146218756866e529 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Mon, 28 Jul 2025 16:46:05 -0700 Subject: [PATCH 13/70] turned off paralle test run --- .github/workflows/smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 6c5c28694..bd06c8c8f 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -90,7 +90,7 @@ jobs: # Start server build_scripts/server_start.sh - if ! pytest -s --verbose -m "smoke" --timer-top-n 100 -n auto; then + if ! pytest -s --verbose -m "smoke" --timer-top-n 100; then echo "====================================================" echo "====================================================" echo "=== Server_Sevice_Agent.LOG ===" From d896f00f57b8368b770e4ec088f7deba6a58f24c Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 09:36:36 -0700 Subject: [PATCH 14/70] added printout debug --- .github/workflows/smoke.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index bd06c8c8f..926cb37cb 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -87,17 +87,23 @@ jobs: aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} EOF - # Start server - build_scripts/server_start.sh - - if ! pytest -s --verbose -m "smoke" --timer-top-n 100; then - echo "====================================================" - echo "====================================================" - echo "=== Server_Sevice_Agent.LOG ===" - cat agent.log || echo "(no agent.log found)" - echo "====================================================" - exit 1 - fi + # Print out the files to verify + echo "--- ~/.aws/config ---" + cat ~/.aws/config + echo "--- ~/.aws/credentials ---" + cat ~/.aws/credentials + + # # Start server + # build_scripts/server_start.sh + + # if ! pytest -s --verbose -m "smoke" --timer-top-n 100; then + # echo "====================================================" + # echo "====================================================" + # echo "=== Server_Sevice_Agent.LOG ===" + # cat agent.log || echo "(no agent.log found)" + # echo "====================================================" + # exit 1 + # fi - name: Notify Slack on success if: success() From c6bacef98d87fb3f083b7cab141c730953f4d2fc Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 09:43:07 -0700 Subject: [PATCH 15/70] debug1 --- .github/workflows/smoke.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 926cb37cb..de0ee2efe 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -93,6 +93,9 @@ jobs: echo "--- ~/.aws/credentials ---" cat ~/.aws/credentials + # debug + echo "OPENAI_API_VERSION = ${OPENAI_API_VERSION}" + # # Start server # build_scripts/server_start.sh From 5bd211fe937c4aceb18763fa5c56fb2f82796ba6 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 09:49:03 -0700 Subject: [PATCH 16/70] debug2 --- .github/workflows/smoke.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index de0ee2efe..e3c90a36f 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -94,7 +94,9 @@ jobs: cat ~/.aws/credentials # debug - echo "OPENAI_API_VERSION = ${OPENAI_API_VERSION}" + echo "OPENAI_API_VERSION = "${OPENAI_API_VERSION}"" + echo "OPENAI_API_KEY = "${OPENAI_API_KEY}"" + echo "AWS_SECRET_ACCESS_KEY = "${AWS_SECRET_ACCESS_KEY}"" # # Start server # build_scripts/server_start.sh From 07cba3007b1afbdb745abd7ca6a26df1d5a1efcc Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 10:06:40 -0700 Subject: [PATCH 17/70] debug3 --- .github/workflows/smoke.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index e3c90a36f..dbc054290 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,6 +51,13 @@ jobs: - name: Show installed packages run: pip freeze + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + - name: Start server & run smoke tests & print server-log if any failure shell: bash env: @@ -60,8 +67,8 @@ jobs: AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AGENT_TOOL_PATH: "./neuro_san/coded_tools" PYTHONPATH: ${{ env.PYTHONPATH }}:. run: | @@ -83,8 +90,8 @@ jobs: # AWS credentials (pulled from env) cat > ~/.aws/credentials < Date: Tue, 29 Jul 2025 10:20:55 -0700 Subject: [PATCH 18/70] debug --- .github/workflows/smoke.yml | 43 ++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index dbc054290..741f97ea6 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,26 +51,20 @@ jobs: - name: Show installed packages run: pip freeze - - name: Configure AWS credentials + - name: Configure AWS credentials & print uses: aws-actions/configure-aws-credentials@v3 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - - - name: Start server & run smoke tests & print server-log if any failure - shell: bash - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} - OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AGENT_TOOL_PATH: "./neuro_san/coded_tools" - PYTHONPATH: ${{ env.PYTHONPATH }}:. + aws-region: ${{ vars.AWS_REGION }} + # anything under `run:` must be at the same indent as `with:` + run: | + echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" + echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" + echo "AWS_REGION = $AWS_REGION" + aws bedrock list-foundation-models --profile daiDev + + - name: Set up AWS config & credentials run: | # Set up AWS config & credentials mkdir -p ~/.aws @@ -100,6 +94,21 @@ jobs: echo "--- ~/.aws/credentials ---" cat ~/.aws/credentials + - name: Start server & run smoke tests & print server-log if any failure + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AGENT_TOOL_PATH: "./neuro_san/coded_tools" + PYTHONPATH: ${{ env.PYTHONPATH }}:. + + # debug echo "OPENAI_API_VERSION = "${OPENAI_API_VERSION}"" echo "OPENAI_API_KEY = "${OPENAI_API_KEY}"" From 91f9ea975f7ee81667e08c568adfab49ab3a816d Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 10:24:54 -0700 Subject: [PATCH 19/70] fixed debug --- .github/workflows/smoke.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 741f97ea6..083eacd24 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -107,8 +107,7 @@ jobs: # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AGENT_TOOL_PATH: "./neuro_san/coded_tools" PYTHONPATH: ${{ env.PYTHONPATH }}:. - - + run: | # debug echo "OPENAI_API_VERSION = "${OPENAI_API_VERSION}"" echo "OPENAI_API_KEY = "${OPENAI_API_KEY}"" From 152255f6b313dfbbadbe66ab4cb895bde0d2d10c Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 10:27:05 -0700 Subject: [PATCH 20/70] fixed debug --- .github/workflows/smoke.yml | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 083eacd24..ff2432598 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,20 +51,31 @@ jobs: - name: Show installed packages run: pip freeze - - name: Configure AWS credentials & print + - name: Configure AWS creds & list Bedrock models uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ vars.AWS_REGION }} - # anything under `run:` must be at the same indent as `with:` run: | echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" echo "AWS_REGION = $AWS_REGION" aws bedrock list-foundation-models --profile daiDev - - - name: Set up AWS config & credentials + + - name: Start server & run smoke tests & print server-log if any failure + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AGENT_TOOL_PATH: "./neuro_san/coded_tools" + PYTHONPATH: ${{ env.PYTHONPATH }}:. run: | # Set up AWS config & credentials mkdir -p ~/.aws @@ -94,20 +105,6 @@ jobs: echo "--- ~/.aws/credentials ---" cat ~/.aws/credentials - - name: Start server & run smoke tests & print server-log if any failure - shell: bash - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} - OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AGENT_TOOL_PATH: "./neuro_san/coded_tools" - PYTHONPATH: ${{ env.PYTHONPATH }}:. - run: | # debug echo "OPENAI_API_VERSION = "${OPENAI_API_VERSION}"" echo "OPENAI_API_KEY = "${OPENAI_API_KEY}"" From 202d28974a737e902073273524385ac275ef6ecf Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 10:35:31 -0700 Subject: [PATCH 21/70] debug --- .github/workflows/smoke.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index ff2432598..bb089d52a 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -48,20 +48,20 @@ jobs: # run: | # build_scripts/set_ollama_model.sh - - name: Show installed packages - run: pip freeze - - - name: Configure AWS creds & list Bedrock models - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - run: | - echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" - echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" - echo "AWS_REGION = $AWS_REGION" - aws bedrock list-foundation-models --profile daiDev + # - name: Show installed packages + # run: pip freeze + + # - name: Configure AWS creds & list Bedrock models + # uses: aws-actions/configure-aws-credentials@v3 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ vars.AWS_REGION }} + # run: | + # echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" + # echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" + # echo "AWS_REGION = $AWS_REGION" + # aws bedrock list-foundation-models --profile daiDev - name: Start server & run smoke tests & print server-log if any failure shell: bash From 2ed1ae40a7656401c15c754c48b400562a9cb3d8 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 10:39:57 -0700 Subject: [PATCH 22/70] debug --- .github/workflows/smoke.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index bb089d52a..3dc5efc5c 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,17 +51,17 @@ jobs: # - name: Show installed packages # run: pip freeze - # - name: Configure AWS creds & list Bedrock models - # uses: aws-actions/configure-aws-credentials@v3 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ vars.AWS_REGION }} - # run: | - # echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" - # echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" - # echo "AWS_REGION = $AWS_REGION" - # aws bedrock list-foundation-models --profile daiDev + - name: Configure AWS creds & list Bedrock models + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + run: | + echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" + echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" + echo "AWS_REGION = $AWS_REGION" + aws bedrock list-foundation-models --profile daiDev - name: Start server & run smoke tests & print server-log if any failure shell: bash From c6f75b9b42488a590898f170b3eabf39d54cfd1e Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 11:02:51 -0700 Subject: [PATCH 23/70] reset got from main --- .github/workflows/smoke.yml | 97 +++++++++---------------------------- 1 file changed, 24 insertions(+), 73 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 3dc5efc5c..33daf0aab 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -27,15 +27,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # - name: Check for NVIDIA GPU - # shell: bash - # run: | - # build_scripts/check_gpu.sh + - name: Check for NVIDIA GPU + shell: bash + run: | + build_scripts/check_gpu.sh - # - name: check available memory - # shell: bash - # run: | - # build_scripts/check_memory.sh + - name: check available memory + shell: bash + run: | + build_scripts/check_memory.sh - name: Install dependencies run: | @@ -43,25 +43,13 @@ jobs: pip install -r requirements-build.txt pip install -r requirements.txt - # - name: Set Ollama model to be used for related test case - # shell: bash - # run: | - # build_scripts/set_ollama_model.sh - - # - name: Show installed packages - # run: pip freeze - - - name: Configure AWS creds & list Bedrock models - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} + - name: Set Ollama model to be used for related test case + shell: bash run: | - echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" - echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" - echo "AWS_REGION = $AWS_REGION" - aws bedrock list-foundation-models --profile daiDev + build_scripts/set_ollama_model.sh + + - name: Show installed packages + run: pip freeze - name: Start server & run smoke tests & print server-log if any failure shell: bash @@ -72,56 +60,19 @@ jobs: AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AGENT_TOOL_PATH: "./neuro_san/coded_tools" PYTHONPATH: ${{ env.PYTHONPATH }}:. run: | - # Set up AWS config & credentials - mkdir -p ~/.aws - - # AWS CLI config with default and daiDev profile - cat > ~/.aws/config < ~/.aws/credentials < Date: Tue, 29 Jul 2025 11:05:29 -0700 Subject: [PATCH 24/70] commented out for debug --- .github/workflows/smoke.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 33daf0aab..7892634b5 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -43,13 +43,13 @@ jobs: pip install -r requirements-build.txt pip install -r requirements.txt - - name: Set Ollama model to be used for related test case - shell: bash - run: | - build_scripts/set_ollama_model.sh + # - name: Set Ollama model to be used for related test case + # shell: bash + # run: | + # build_scripts/set_ollama_model.sh - - name: Show installed packages - run: pip freeze + # - name: Show installed packages + # run: pip freeze - name: Start server & run smoke tests & print server-log if any failure shell: bash @@ -63,16 +63,16 @@ jobs: AGENT_TOOL_PATH: "./neuro_san/coded_tools" PYTHONPATH: ${{ env.PYTHONPATH }}:. run: | - build_scripts/server_start.sh + # build_scripts/server_start.sh - if ! pytest -s --verbose -m "smoke" --timer-top-n 100 -n auto; then - echo "====================================================" - echo "====================================================" - echo "=== Server_Sevice_Agent.LOG ===" - cat agent.log || echo "(no agent.log found)" - echo "====================================================" - exit 1 - fi + # if ! pytest -s --verbose -m "smoke" --timer-top-n 100 -n auto; then + # echo "====================================================" + # echo "====================================================" + # echo "=== Server_Sevice_Agent.LOG ===" + # cat agent.log || echo "(no agent.log found)" + # echo "====================================================" + # exit 1 + # fi - name: Notify Slack on success if: success() From 3bea3d1a93c569b4437a99c5c51bb427bc269cf1 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 11:11:55 -0700 Subject: [PATCH 25/70] added config aws --- .github/workflows/smoke.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 7892634b5..f93a9e74e 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,6 +51,18 @@ jobs: # - name: Show installed packages # run: pip freeze + - name: Configure AWS creds & list Bedrock models + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + run: | + echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" + echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" + echo "AWS_REGION = $AWS_REGION" + aws bedrock list-foundation-models --profile daiDev + - name: Start server & run smoke tests & print server-log if any failure shell: bash env: From c3d097e0c737ef5bbe7b3be06949291ce16459a5 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 11:17:36 -0700 Subject: [PATCH 26/70] =?UTF-8?q?fixed=20GitHub=20Actions=20doesn=E2=80=99?= =?UTF-8?q?t=20let=20you=20combine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/smoke.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index f93a9e74e..9be243d26 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,17 +51,19 @@ jobs: # - name: Show installed packages # run: pip freeze - - name: Configure AWS creds & list Bedrock models - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - run: | - echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" - echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" - echo "AWS_REGION = $AWS_REGION" - aws bedrock list-foundation-models --profile daiDev + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Verify creds & list Bedrock models + run: | + echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" + echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" + echo "AWS_REGION = $AWS_REGION" + aws bedrock list-foundation-models --profile daiDev - name: Start server & run smoke tests & print server-log if any failure shell: bash From d372bd7f14e1ebeaf4402978bac28c5ee7877cb7 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 11:19:59 -0700 Subject: [PATCH 27/70] fixed indent --- .github/workflows/smoke.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 9be243d26..dd88c0566 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,19 +51,19 @@ jobs: # - name: Show installed packages # run: pip freeze - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_REGION }} - - name: Verify creds & list Bedrock models - run: | - echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" - echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" - echo "AWS_REGION = $AWS_REGION" - aws bedrock list-foundation-models --profile daiDev + - name: Verify creds & list Bedrock models + run: | + echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" + echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" + echo "AWS_REGION = $AWS_REGION" + aws bedrock list-foundation-models --profile daiDev - name: Start server & run smoke tests & print server-log if any failure shell: bash From 1918b4acd778af3490edc9be16cc9c99936b4cea Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 11:36:44 -0700 Subject: [PATCH 28/70] debug to see list model work or not --- .github/workflows/smoke.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index dd88c0566..2d9094d3c 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -56,13 +56,27 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} + # aws-region: ${{ vars.AWS_REGION }} - name: Verify creds & list Bedrock models run: | + mkdir -p ~/.aws + + # AWS CLI config with default and daiDev profile + cat > ~/.aws/config < Date: Tue, 29 Jul 2025 11:42:25 -0700 Subject: [PATCH 29/70] debug add more aws stuff needed --- .github/workflows/smoke.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 2d9094d3c..a3bb15cf8 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,12 +51,27 @@ jobs: # - name: Show installed packages # run: pip freeze + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up variables + id: set-vars + run: | + if [ "${{ github.event_name }}" = "release" ]; then + echo "image_tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT + echo "target_environment=staging" >> $GITHUB_OUTPUT + else + SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) + echo "image_tag=${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "target_environment=dev" >> $GITHUB_OUTPUT + fi + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: ${{ vars.AWS_REGION }} + aws-region: ${{ vars.AWS_REGION }} - name: Verify creds & list Bedrock models run: | @@ -76,7 +91,7 @@ jobs: echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" - # echo "AWS_REGION = $AWS_REGION" + echo "AWS_REGION = $AWS_REGION" aws bedrock list-foundation-models --profile daiDev - name: Start server & run smoke tests & print server-log if any failure From b17e69360735c37dcb4d1750fbe71a7031f05225 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 11:47:21 -0700 Subject: [PATCH 30/70] debug added region value --- .github/workflows/smoke.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index a3bb15cf8..bdd7942b7 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -50,28 +50,13 @@ jobs: # - name: Show installed packages # run: pip freeze - - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up variables - id: set-vars - run: | - if [ "${{ github.event_name }}" = "release" ]; then - echo "image_tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT - echo "target_environment=staging" >> $GITHUB_OUTPUT - else - SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) - echo "image_tag=${SHORT_SHA}" >> $GITHUB_OUTPUT - echo "target_environment=dev" >> $GITHUB_OUTPUT - fi - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} + aws-region: us-west-2 - name: Verify creds & list Bedrock models run: | From 3c96a3e84bae7c47561e0fb356c45122f353c499 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 11:56:03 -0700 Subject: [PATCH 31/70] debug still --- .github/workflows/smoke.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index bdd7942b7..0c38b7ae3 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -54,9 +54,12 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v3 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 + # Required credentials or OIDC settings: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds + audience: sts.amazonaws.com # if you’re assuming via OIDC + # 🔑 THIS IS MANDATORY: + aws-region: ${{ vars.AWS_REGION }} - name: Verify creds & list Bedrock models run: | From df6b951d5912a78ca92c595fd26d8306fdfd4606 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 12:02:39 -0700 Subject: [PATCH 32/70] debug Make sure AWS_REGION is set --- .github/workflows/smoke.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 0c38b7ae3..938fc13c0 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -8,6 +8,10 @@ on: jobs: smoke-test: runs-on: ubuntu-latest + + # Make sure AWS_REGION is set, e.g. in your repo vars or here: + env: + AWS_REGION: us-west-2 # 🚀 Ollama Service runs alongside main container services: @@ -57,9 +61,8 @@ jobs: # Required credentials or OIDC settings: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds - audience: sts.amazonaws.com # if you’re assuming via OIDC # 🔑 THIS IS MANDATORY: - aws-region: ${{ vars.AWS_REGION }} + aws-region: ${{ env.AWS_REGION }} - name: Verify creds & list Bedrock models run: | From b1ba3e57a1e3d7506d0e8f831bfb792eced113a7 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 12:13:50 -0700 Subject: [PATCH 33/70] debug add role --- .github/workflows/smoke.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 938fc13c0..ccc5f6be1 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -12,6 +12,7 @@ jobs: # Make sure AWS_REGION is set, e.g. in your repo vars or here: env: AWS_REGION: us-west-2 + AWS_ROLL: arn:aws:iam::613076437200:role/LeafDev # 🚀 Ollama Service runs alongside main container services: @@ -62,6 +63,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds # 🔑 THIS IS MANDATORY: + role-to-assume: ${{ env.AWS_ROLL }} aws-region: ${{ env.AWS_REGION }} - name: Verify creds & list Bedrock models From a33e694aa739335394112917d2936df215462604 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 12:22:53 -0700 Subject: [PATCH 34/70] debug added permission --- .github/workflows/smoke.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index ccc5f6be1..0f3956d6a 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -5,6 +5,11 @@ on: - cron: '0 12 * * *' # 12:00 PM UTC = 4:00 AM PT (standard) workflow_dispatch: # enables manual triggering +# 1) Grant the workflow OIDC permission +permissions: + id-token: write + contents: read + jobs: smoke-test: runs-on: ubuntu-latest From db63745356842677711fe2fee71c2f6f1441a0f0 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 12:45:37 -0700 Subject: [PATCH 35/70] debug --- .github/workflows/smoke.yml | 55 +++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 0f3956d6a..91928c309 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -5,10 +5,10 @@ on: - cron: '0 12 * * *' # 12:00 PM UTC = 4:00 AM PT (standard) workflow_dispatch: # enables manual triggering -# 1) Grant the workflow OIDC permission -permissions: - id-token: write - contents: read +# # 1) Grant the workflow OIDC permission +# permissions: +# id-token: write +# contents: read jobs: smoke-test: @@ -18,6 +18,14 @@ jobs: env: AWS_REGION: us-west-2 AWS_ROLL: arn:aws:iam::613076437200:role/LeafDev + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds + + run: | + echo "AWS-ACCESS-KEY-ID = $AWS-ACCESS-KEY-ID" + echo "AWS-SECRET-ACCESS-KEY = $AWS-SECRET-ACCESS-KEY" + echo "aws-access-key-id = $aws-access-key-id" + echo "aws-secret-access-key = $aws-secret-access-key" # 🚀 Ollama Service runs alongside main container services: @@ -61,15 +69,15 @@ jobs: # - name: Show installed packages # run: pip freeze - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - # Required credentials or OIDC settings: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds - # 🔑 THIS IS MANDATORY: - role-to-assume: ${{ env.AWS_ROLL }} - aws-region: ${{ env.AWS_REGION }} + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v3 + # with: + # # Required credentials or OIDC settings: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds + # # 🔑 THIS IS MANDATORY: + # role-to-assume: ${{ env.AWS_ROLL }} + # aws-region: ${{ env.AWS_REGION }} - name: Verify creds & list Bedrock models run: | @@ -87,10 +95,23 @@ jobs: source_profile=default EOF - echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" - echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" - echo "AWS_REGION = $AWS_REGION" - aws bedrock list-foundation-models --profile daiDev + echo "AWS-ACCESS-KEY-ID = $AWS-ACCESS-KEY-ID" + echo "AWS-SECRET-ACCESS-KEY = $AWS-SECRET-ACCESS-KEY" + + # AWS credentials (pulled from env) + cat > ~/.aws/credentials < Date: Tue, 29 Jul 2025 12:49:00 -0700 Subject: [PATCH 36/70] debug --- .github/workflows/smoke.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 91928c309..a2e974b01 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -20,7 +20,8 @@ jobs: AWS_ROLL: arn:aws:iam::613076437200:role/LeafDev aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds - + + name: Print Debug Env values run: | echo "AWS-ACCESS-KEY-ID = $AWS-ACCESS-KEY-ID" echo "AWS-SECRET-ACCESS-KEY = $AWS-SECRET-ACCESS-KEY" From 8e7b421843940983e1b1b74c2ef6d2533ad80bb8 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 12:50:07 -0700 Subject: [PATCH 37/70] debug failure --- .github/workflows/smoke.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index a2e974b01..e64a915ad 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -21,8 +21,8 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds - name: Print Debug Env values - run: | + - name: Print Debug Env values + run: | echo "AWS-ACCESS-KEY-ID = $AWS-ACCESS-KEY-ID" echo "AWS-SECRET-ACCESS-KEY = $AWS-SECRET-ACCESS-KEY" echo "aws-access-key-id = $aws-access-key-id" From 268d307b6f5533afd9dbccf9654449915ad48bc6 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 12:52:09 -0700 Subject: [PATCH 38/70] removed echo --- .github/workflows/smoke.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index e64a915ad..bd0e7d4fd 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -21,13 +21,6 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds - - name: Print Debug Env values - run: | - echo "AWS-ACCESS-KEY-ID = $AWS-ACCESS-KEY-ID" - echo "AWS-SECRET-ACCESS-KEY = $AWS-SECRET-ACCESS-KEY" - echo "aws-access-key-id = $aws-access-key-id" - echo "aws-secret-access-key = $aws-secret-access-key" - # 🚀 Ollama Service runs alongside main container services: ollama: @@ -98,6 +91,8 @@ jobs: echo "AWS-ACCESS-KEY-ID = $AWS-ACCESS-KEY-ID" echo "AWS-SECRET-ACCESS-KEY = $AWS-SECRET-ACCESS-KEY" + echo "aws-access-key-id = $aws-access-key-id" + echo "aws-secret-access-key = $aws-secret-access-key" # AWS credentials (pulled from env) cat > ~/.aws/credentials < Date: Tue, 29 Jul 2025 12:56:37 -0700 Subject: [PATCH 39/70] reversed some changes --- .github/workflows/smoke.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index bd0e7d4fd..9c00e0d56 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -16,10 +16,9 @@ jobs: # Make sure AWS_REGION is set, e.g. in your repo vars or here: env: - AWS_REGION: us-west-2 - AWS_ROLL: arn:aws:iam::613076437200:role/LeafDev - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds + # 🚀 Ollama Service runs alongside main container services: @@ -89,16 +88,14 @@ jobs: source_profile=default EOF - echo "AWS-ACCESS-KEY-ID = $AWS-ACCESS-KEY-ID" - echo "AWS-SECRET-ACCESS-KEY = $AWS-SECRET-ACCESS-KEY" - echo "aws-access-key-id = $aws-access-key-id" - echo "aws-secret-access-key = $aws-secret-access-key" + echo "AWS_ACCESS_KEY_ID = $AWS_ACCESS_KEY_ID" + echo "AWS_SECRET_ACCESS_KEY = $AWS_SECRET_ACCESS_KEY" # AWS credentials (pulled from env) cat > ~/.aws/credentials < Date: Tue, 29 Jul 2025 12:59:30 -0700 Subject: [PATCH 40/70] reversed to go back --- .github/workflows/smoke.yml | 87 +++++++------------------------------ 1 file changed, 16 insertions(+), 71 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 9c00e0d56..33daf0aab 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -5,20 +5,9 @@ on: - cron: '0 12 * * *' # 12:00 PM UTC = 4:00 AM PT (standard) workflow_dispatch: # enables manual triggering -# # 1) Grant the workflow OIDC permission -# permissions: -# id-token: write -# contents: read - jobs: smoke-test: runs-on: ubuntu-latest - - # Make sure AWS_REGION is set, e.g. in your repo vars or here: - env: - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds - # 🚀 Ollama Service runs alongside main container services: @@ -54,57 +43,13 @@ jobs: pip install -r requirements-build.txt pip install -r requirements.txt - # - name: Set Ollama model to be used for related test case - # shell: bash - # run: | - # build_scripts/set_ollama_model.sh - - # - name: Show installed packages - # run: pip freeze - - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v3 - # with: - # # Required credentials or OIDC settings: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} # if using static creds - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # if using static creds - # # 🔑 THIS IS MANDATORY: - # role-to-assume: ${{ env.AWS_ROLL }} - # aws-region: ${{ env.AWS_REGION }} - - - name: Verify creds & list Bedrock models + - name: Set Ollama model to be used for related test case + shell: bash run: | - mkdir -p ~/.aws - - # AWS CLI config with default and daiDev profile - cat > ~/.aws/config < ~/.aws/credentials < Date: Tue, 29 Jul 2025 13:10:17 -0700 Subject: [PATCH 41/70] reset debug at the beginning --- .github/workflows/smoke.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 33daf0aab..4b38890ec 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,6 +51,43 @@ jobs: - name: Show installed packages run: pip freeze + name: Set up AWS config & credentials + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + mkdir -p ~/.aws + + # AWS CLI config with default and daiDev profile + cat > ~/.aws/config < ~/.aws/credentials < Date: Tue, 29 Jul 2025 13:11:19 -0700 Subject: [PATCH 42/70] fixed typo --- .github/workflows/smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 4b38890ec..a5a932890 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,7 +51,7 @@ jobs: - name: Show installed packages run: pip freeze - name: Set up AWS config & credentials + - name: Set up AWS config & credentials env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From d335befe678d12e05f9827061c90d76a59f87610 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 13:20:53 -0700 Subject: [PATCH 43/70] debug 1 --- .github/workflows/smoke.yml | 63 +++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index a5a932890..450501085 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -43,13 +43,13 @@ jobs: pip install -r requirements-build.txt pip install -r requirements.txt - - name: Set Ollama model to be used for related test case - shell: bash - run: | - build_scripts/set_ollama_model.sh + # - name: Set Ollama model to be used for related test case + # shell: bash + # run: | + # build_scripts/set_ollama_model.sh - - name: Show installed packages - run: pip freeze + # - name: Show installed packages + # run: pip freeze - name: Set up AWS config & credentials env: @@ -65,7 +65,7 @@ jobs: output=json [profile daiDev] - role_arn=arn:aws:iam::111111:role/testonly + role_arn=arn:aws:iam::613076437200:role/LeafDev region=us-west-2 source_profile=default EOF @@ -87,29 +87,32 @@ jobs: cat ~/.aws/config echo "--- ~/.aws/credentials ---" cat ~/.aws/credentials - - - name: Start server & run smoke tests & print server-log if any failure - shell: bash - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} - OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} - AGENT_TOOL_PATH: "./neuro_san/coded_tools" - PYTHONPATH: ${{ env.PYTHONPATH }}:. - run: | - build_scripts/server_start.sh - - if ! pytest -s --verbose -m "smoke" --timer-top-n 100 -n auto; then - echo "====================================================" - echo "====================================================" - echo "=== Server_Sevice_Agent.LOG ===" - cat agent.log || echo "(no agent.log found)" - echo "====================================================" - exit 1 - fi + + - name: List Bedrock foundation models + run: aws bedrock list-foundation-models --profile daiDev + + # - name: Start server & run smoke tests & print server-log if any failure + # shell: bash + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + # AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + # AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + # OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} + # AGENT_TOOL_PATH: "./neuro_san/coded_tools" + # PYTHONPATH: ${{ env.PYTHONPATH }}:. + # run: | + # build_scripts/server_start.sh + + # if ! pytest -s --verbose -m "smoke" --timer-top-n 100 -n auto; then + # echo "====================================================" + # echo "====================================================" + # echo "=== Server_Sevice_Agent.LOG ===" + # cat agent.log || echo "(no agent.log found)" + # echo "====================================================" + # exit 1 + # fi - name: Notify Slack on success if: success() From 36034563b95247447bd699ad0a1bb8fa149c3df4 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 13:25:04 -0700 Subject: [PATCH 44/70] debug install aws cli --- .github/workflows/smoke.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 450501085..3337794eb 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,6 +51,14 @@ jobs: # - name: Show installed packages # run: pip freeze + # 1) Install AWS CLI v2 (which includes the Bedrock commands) + - name: Install AWS CLI v2 + run: | + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscliv2.zip + unzip awscliv2.zip + sudo ./aws/install + aws --version + - name: Set up AWS config & credentials env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} From a093cc32c0513243d906e888b13d2203cbc2f9bf Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 13:29:09 -0700 Subject: [PATCH 45/70] Debug add unzip --- .github/workflows/smoke.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 3337794eb..913fa5444 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -51,6 +51,11 @@ jobs: # - name: Show installed packages # run: pip freeze + - name: Install AWS CLI prerequisites + run: | + sudo apt-get update + sudo apt-get install -y unzip zip + # 1) Install AWS CLI v2 (which includes the Bedrock commands) - name: Install AWS CLI v2 run: | @@ -58,7 +63,7 @@ jobs: unzip awscliv2.zip sudo ./aws/install aws --version - + - name: Set up AWS config & credentials env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} From e6551d6c51c6c24a3a0b7edcb467006b0922fe8b Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 13:32:04 -0700 Subject: [PATCH 46/70] debug removed as not needed --- .github/workflows/smoke.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 913fa5444..36963d50a 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -53,8 +53,8 @@ jobs: - name: Install AWS CLI prerequisites run: | - sudo apt-get update - sudo apt-get install -y unzip zip + apt-get update + apt-get install -y unzip zip # 1) Install AWS CLI v2 (which includes the Bedrock commands) - name: Install AWS CLI v2 From bd8d5347596ae4e1d1ed40ef48b51e067f3e7cf8 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 13:36:57 -0700 Subject: [PATCH 47/70] debug removed more sudo --- .github/workflows/smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 36963d50a..7ebc39fb5 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -61,7 +61,7 @@ jobs: run: | curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscliv2.zip unzip awscliv2.zip - sudo ./aws/install + ./aws/install aws --version - name: Set up AWS config & credentials From 38a72e56670bfceb4e9e3ac8cd4cdda2c29c0329 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 13:47:02 -0700 Subject: [PATCH 48/70] debug fix indention --- .github/workflows/smoke.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 7ebc39fb5..c8c8fa078 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -73,22 +73,22 @@ jobs: # AWS CLI config with default and daiDev profile cat > ~/.aws/config < ~/.aws/credentials < Date: Tue, 29 Jul 2025 13:50:30 -0700 Subject: [PATCH 49/70] Debug another fixed indent --- .github/workflows/smoke.yml | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index c8c8fa078..1f8f2cd49 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -71,31 +71,20 @@ jobs: run: | mkdir -p ~/.aws - # AWS CLI config with default and daiDev profile - cat > ~/.aws/config < ~/.aws/config # AWS credentials (pulled from env) - cat > ~/.aws/credentials < ~/.aws/credentials - # Tighten permissions chmod 700 ~/.aws - chmod 600 ~/.aws/config - chmod 600 ~/.aws/credentials + chmod 600 ~/.aws/config ~/.aws/credentials - # Print out the files to verify echo "--- ~/.aws/config ---" cat ~/.aws/config echo "--- ~/.aws/credentials ---" From 18e51a2f2f42261bdb40fd1bfa982cb6fa939f80 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 14:03:46 -0700 Subject: [PATCH 50/70] Tweaked to created aws files --- .github/workflows/smoke.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 1f8f2cd49..bbc0a38b2 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -71,24 +71,24 @@ jobs: run: | mkdir -p ~/.aws - # AWS CLI config - printf "[default]\nregion=us-west-2\noutput=json\n\n" \ - "[profile daiDev]\nrole_arn=arn:aws:iam::613076437200:role/LeafDev\n" \ - "region=us-west-2\nsource_profile=default\n" \ + # Print to verify the vars are in the environment + echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" + + # Write config + printf "[default]\nregion=%s\noutput=json\n\n[profile daiDev]\nrole_arn=arn:aws:iam::613076437200:role/LeafDev\nregion=%s\nsource_profile=default\n" \ + "$AWS_REGION" "$AWS_REGION" \ > ~/.aws/config - # AWS credentials (pulled from env) + # Write credentials printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ - "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ + "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ > ~/.aws/credentials chmod 700 ~/.aws chmod 600 ~/.aws/config ~/.aws/credentials - echo "--- ~/.aws/config ---" - cat ~/.aws/config - echo "--- ~/.aws/credentials ---" - cat ~/.aws/credentials + echo "--- ~/.aws/config ---" && cat ~/.aws/config + echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials - name: List Bedrock foundation models From 6b74e96325bb0e48f8f557d0671587ac5c010e02 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 14:15:33 -0700 Subject: [PATCH 51/70] added region --- .github/workflows/smoke.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index bbc0a38b2..406320386 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -68,6 +68,8 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: us-west-2 + run: | mkdir -p ~/.aws From 581837642fba1ce5e59f1b8c72997f8ab377ad4e Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 14:23:37 -0700 Subject: [PATCH 52/70] moved env to top --- .github/workflows/smoke.yml | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 406320386..49cd6a41b 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -8,6 +8,13 @@ on: jobs: smoke-test: runs-on: ubuntu-latest + + # ─── Job‑level ENV ───────────────────────────────────────── + # every step below will have these in its shell environment + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: us-west-2 # 🚀 Ollama Service runs alongside main container services: @@ -65,23 +72,21 @@ jobs: aws --version - name: Set up AWS config & credentials - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 - run: | mkdir -p ~/.aws - # Print to verify the vars are in the environment - echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" + # sanity check that the JOB‑LEVEL env is visible + echo "ENV CHECK: AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" + echo "ENV CHECK: AWS_REGION=$AWS_REGION" - # Write config - printf "[default]\nregion=%s\noutput=json\n\n[profile daiDev]\nrole_arn=arn:aws:iam::613076437200:role/LeafDev\nregion=%s\nsource_profile=default\n" \ - "$AWS_REGION" "$AWS_REGION" \ + # write config + printf "[default]\nregion=%s\noutput=json\n\n[profile daiDev]\n" \ + "role_arn=arn:aws:iam::613076437200:role/LeafDev\n" \ + "region=%s\nsource_profile=default\n" \ + "$AWS_REGION" "$AWS_REGION" \ > ~/.aws/config - # Write credentials + # write credentials printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ > ~/.aws/credentials From f066f9146c93a6117a036edbb09b6ed6f0d32910 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 14:28:26 -0700 Subject: [PATCH 53/70] Tweaked --- .github/workflows/smoke.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 49cd6a41b..351b688b1 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -75,18 +75,15 @@ jobs: run: | mkdir -p ~/.aws - # sanity check that the JOB‑LEVEL env is visible - echo "ENV CHECK: AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" - echo "ENV CHECK: AWS_REGION=$AWS_REGION" - - # write config - printf "[default]\nregion=%s\noutput=json\n\n[profile daiDev]\n" \ - "role_arn=arn:aws:iam::613076437200:role/LeafDev\n" \ - "region=%s\nsource_profile=default\n" \ - "$AWS_REGION" "$AWS_REGION" \ + # Print to verify the vars are in the environment + echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" + + # Write config + printf "[default]\nregion=%s\noutput=json\n\n[profile daiDev]\nrole_arn=arn:aws:iam::613076437200:role/LeafDev\nregion=%s\nsource_profile=default\n" \ + "$AWS_REGION" "$AWS_REGION" \ > ~/.aws/config - # write credentials + # Write credentials printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ > ~/.aws/credentials From 58b8b86c4a1e92b008b276be5afbb34e38454abc Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 15:07:51 -0700 Subject: [PATCH 54/70] Set to use variables GitHub Action --- .github/workflows/smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 351b688b1..477a379be 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -14,7 +14,7 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-west-2 + AWS_REGION: ${{ vars.AWS_REGION }} # 🚀 Ollama Service runs alongside main container services: From a189a04d065d3718c669427dd24841cbca5f5c05 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:10:31 -0700 Subject: [PATCH 55/70] commented out AWS env --- .github/workflows/smoke.yml | 46 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 477a379be..42a70c0ca 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -9,12 +9,12 @@ jobs: smoke-test: runs-on: ubuntu-latest - # ─── Job‑level ENV ───────────────────────────────────────── - # every step below will have these in its shell environment - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ vars.AWS_REGION }} + # # ─── Job‑level ENV ───────────────────────────────────────── + # # every step below will have these in its shell environment + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # AWS_REGION: ${{ vars.AWS_REGION }} # 🚀 Ollama Service runs alongside main container services: @@ -71,28 +71,28 @@ jobs: ./aws/install aws --version - - name: Set up AWS config & credentials - run: | - mkdir -p ~/.aws + # - name: Set up AWS config & credentials + # run: | + # mkdir -p ~/.aws - # Print to verify the vars are in the environment - echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" + # # Print to verify the vars are in the environment + # echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" - # Write config - printf "[default]\nregion=%s\noutput=json\n\n[profile daiDev]\nrole_arn=arn:aws:iam::613076437200:role/LeafDev\nregion=%s\nsource_profile=default\n" \ - "$AWS_REGION" "$AWS_REGION" \ - > ~/.aws/config + # # Write config + # printf "[default]\nregion=%s\noutput=json\n\n[profile daiDev]\nrole_arn=arn:aws:iam::613076437200:role/LeafDev\nregion=%s\nsource_profile=default\n" \ + # "$AWS_REGION" "$AWS_REGION" \ + # > ~/.aws/config - # Write credentials - printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ - "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ - > ~/.aws/credentials + # # Write credentials + # printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ + # "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ + # > ~/.aws/credentials - chmod 700 ~/.aws - chmod 600 ~/.aws/config ~/.aws/credentials + # chmod 700 ~/.aws + # chmod 600 ~/.aws/config ~/.aws/credentials - echo "--- ~/.aws/config ---" && cat ~/.aws/config - echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials + # echo "--- ~/.aws/config ---" && cat ~/.aws/config + # echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials - name: List Bedrock foundation models From 5e3e487c748fa0692bed98c59c2d5fa28633c481 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:14:13 -0700 Subject: [PATCH 56/70] removed aws check profile --- .github/workflows/smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 42a70c0ca..a55e423e2 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -96,7 +96,7 @@ jobs: - name: List Bedrock foundation models - run: aws bedrock list-foundation-models --profile daiDev + run: aws bedrock list-foundation-models # - name: Start server & run smoke tests & print server-log if any failure # shell: bash From d071dcecede46c9e8795087b4bc984f01f2b08df Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:20:18 -0700 Subject: [PATCH 57/70] Need set region --- .github/workflows/smoke.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index a55e423e2..f3d242a70 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -9,12 +9,12 @@ jobs: smoke-test: runs-on: ubuntu-latest - # # ─── Job‑level ENV ───────────────────────────────────────── - # # every step below will have these in its shell environment - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # AWS_REGION: ${{ vars.AWS_REGION }} + # ─── Job‑level ENV ───────────────────────────────────────── + # every step below will have these in its shell environment + env: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ vars.AWS_REGION }} # 🚀 Ollama Service runs alongside main container services: @@ -71,27 +71,27 @@ jobs: ./aws/install aws --version - # - name: Set up AWS config & credentials - # run: | - # mkdir -p ~/.aws + - name: Set up AWS config & credentials + run: | + mkdir -p ~/.aws - # # Print to verify the vars are in the environment - # echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" + # Print to verify the vars are in the environment + echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" - # # Write config - # printf "[default]\nregion=%s\noutput=json\n\n[profile daiDev]\nrole_arn=arn:aws:iam::613076437200:role/LeafDev\nregion=%s\nsource_profile=default\n" \ - # "$AWS_REGION" "$AWS_REGION" \ - # > ~/.aws/config + # Write config + printf "[default]\nregion=%s\noutput=json" \ + "$AWS_REGION" "$AWS_REGION" \ + > ~/.aws/config # # Write credentials # printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ # "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ # > ~/.aws/credentials - # chmod 700 ~/.aws - # chmod 600 ~/.aws/config ~/.aws/credentials + chmod 700 ~/.aws + chmod 600 ~/.aws/config - # echo "--- ~/.aws/config ---" && cat ~/.aws/config + echo "--- ~/.aws/config ---" && cat ~/.aws/config # echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials From 9d23d9c8836957f0039401d8b95a68c06e124b56 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:24:01 -0700 Subject: [PATCH 58/70] fixed broken yml --- .github/workflows/smoke.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index f3d242a70..86c53a2c0 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -14,7 +14,7 @@ jobs: env: # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ vars.AWS_REGION }} + AWS_REGION: ${{ vars.AWS_REGIO }} # 🚀 Ollama Service runs alongside main container services: @@ -75,28 +75,29 @@ jobs: run: | mkdir -p ~/.aws - # Print to verify the vars are in the environment - echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" + # # Print to verify the vars are in the environment + # echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" # Write config - printf "[default]\nregion=%s\noutput=json" \ + printf "[default]\nregion=%s\noutput=json\n" \ "$AWS_REGION" "$AWS_REGION" \ > ~/.aws/config - # # Write credentials - # printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ - # "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ - # > ~/.aws/credentials + # # Write credentials + # printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ + # "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ + # > ~/.aws/credentials chmod 700 ~/.aws - chmod 600 ~/.aws/config + chmod 600 ~/.aws/config + # chmod 600 ~/.aws/credentials echo "--- ~/.aws/config ---" && cat ~/.aws/config - # echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials + # echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials - name: List Bedrock foundation models - run: aws bedrock list-foundation-models + run: aws bedrock list-foundation-models --profile daiDev # - name: Start server & run smoke tests & print server-log if any failure # shell: bash From cccf5211201855b858806669e1925949e2335fe5 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:30:43 -0700 Subject: [PATCH 59/70] Tweaked --- .github/workflows/smoke.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 86c53a2c0..90efb0a2f 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -14,7 +14,7 @@ jobs: env: # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ vars.AWS_REGIO }} + AWS_REGION: ${{ vars.AWS_REGION }} # 🚀 Ollama Service runs alongside main container services: @@ -80,7 +80,7 @@ jobs: # Write config printf "[default]\nregion=%s\noutput=json\n" \ - "$AWS_REGION" "$AWS_REGION" \ + "$AWS_REGION" \ > ~/.aws/config # # Write credentials From b7b637877c2dc3ce53f97eceab1a3bc2a8836026 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:33:46 -0700 Subject: [PATCH 60/70] removed daiDev --- .github/workflows/smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 90efb0a2f..88520079d 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -97,7 +97,7 @@ jobs: - name: List Bedrock foundation models - run: aws bedrock list-foundation-models --profile daiDev + run: aws bedrock list-foundation-models # - name: Start server & run smoke tests & print server-log if any failure # shell: bash From d2516e41061555d8442dc247155d0b305a4ab025 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:37:25 -0700 Subject: [PATCH 61/70] Set credentials --- .github/workflows/smoke.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 88520079d..b73f8c1dd 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -12,8 +12,8 @@ jobs: # ─── Job‑level ENV ───────────────────────────────────────── # every step below will have these in its shell environment env: - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ vars.AWS_REGION }} # 🚀 Ollama Service runs alongside main container From 8e8cf4988cbd1cabb1b7bf89189562dff704d32b Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:40:27 -0700 Subject: [PATCH 62/70] re-enable test --- .github/workflows/smoke.yml | 44 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index b73f8c1dd..14b2bcf4e 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -99,28 +99,28 @@ jobs: - name: List Bedrock foundation models run: aws bedrock list-foundation-models - # - name: Start server & run smoke tests & print server-log if any failure - # shell: bash - # env: - # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - # ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - # GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - # AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - # AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} - # OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} - # AGENT_TOOL_PATH: "./neuro_san/coded_tools" - # PYTHONPATH: ${{ env.PYTHONPATH }}:. - # run: | - # build_scripts/server_start.sh - - # if ! pytest -s --verbose -m "smoke" --timer-top-n 100 -n auto; then - # echo "====================================================" - # echo "====================================================" - # echo "=== Server_Sevice_Agent.LOG ===" - # cat agent.log || echo "(no agent.log found)" - # echo "====================================================" - # exit 1 - # fi + - name: Start server & run smoke tests & print server-log if any failure + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} + OPENAI_API_VERSION: ${{ vars.OPENAI_API_VERSION }} + AGENT_TOOL_PATH: "./neuro_san/coded_tools" + PYTHONPATH: ${{ env.PYTHONPATH }}:. + run: | + build_scripts/server_start.sh + + if ! pytest -s --verbose -m "smoke" --timer-top-n 100 -n auto; then + echo "====================================================" + echo "====================================================" + echo "=== Server_Sevice_Agent.LOG ===" + cat agent.log || echo "(no agent.log found)" + echo "====================================================" + exit 1 + fi - name: Notify Slack on success if: success() From d3fe8f1110dab12e4ef57a123de5c5f3312d1d27 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 16:53:47 -0700 Subject: [PATCH 63/70] removed bedrock run with multiple profile --- neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon | 1 - tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon b/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon index bad37b0a5..67bdca108 100644 --- a/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon +++ b/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon @@ -16,7 +16,6 @@ { "llm_config": { "model_name": "bedrock-us-claude-3-7-sonnet", - "credentials_profile_name": "daiDev", "region_name": "us-west-2" }, "tools": [ diff --git a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py index 536af7a60..6a3807aad 100644 --- a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py +++ b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py @@ -91,7 +91,7 @@ class TestSmokeTestHocons(TestCase): # List more hocon files as they become available here. ])) - @pytest.mark.timeout(60) # in seconds for this test + @pytest.mark.timeout(5 * 60) # in seconds for this test @pytest.mark.smoke @pytest.mark.non_default_llm_provider @pytest.mark.anthropic From 242be261a521f740d1f4b49b3e40219e894330f3 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 17:07:26 -0700 Subject: [PATCH 64/70] Clean up --- .github/workflows/smoke.yml | 36 +++++++++---------- .../zzz_hocons/test_smoke_test_hocons.py | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 14b2bcf4e..d0cf6ca41 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -71,29 +71,29 @@ jobs: ./aws/install aws --version - - name: Set up AWS config & credentials - run: | - mkdir -p ~/.aws + # - name: Set up AWS config & credentials + # run: | + # mkdir -p ~/.aws - # # Print to verify the vars are in the environment - # echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" + # # Print to verify the vars are in the environment + # echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" - # Write config - printf "[default]\nregion=%s\noutput=json\n" \ - "$AWS_REGION" \ - > ~/.aws/config + # # Write config + # printf "[default]\nregion=%s\noutput=json\n" \ + # "$AWS_REGION" \ + # > ~/.aws/config - # # Write credentials - # printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ - # "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ - # > ~/.aws/credentials + # # Write credentials + # printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ + # "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ + # > ~/.aws/credentials - chmod 700 ~/.aws - chmod 600 ~/.aws/config - # chmod 600 ~/.aws/credentials + # chmod 700 ~/.aws + # chmod 600 ~/.aws/config + # chmod 600 ~/.aws/credentials - echo "--- ~/.aws/config ---" && cat ~/.aws/config - # echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials + # echo "--- ~/.aws/config ---" && cat ~/.aws/config + # echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials - name: List Bedrock foundation models diff --git a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py index 6a3807aad..536af7a60 100644 --- a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py +++ b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py @@ -91,7 +91,7 @@ class TestSmokeTestHocons(TestCase): # List more hocon files as they become available here. ])) - @pytest.mark.timeout(5 * 60) # in seconds for this test + @pytest.mark.timeout(60) # in seconds for this test @pytest.mark.smoke @pytest.mark.non_default_llm_provider @pytest.mark.anthropic From d7f9d78f7a00b59a75d02cfc4d42ea515f3e2026 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 17:20:37 -0700 Subject: [PATCH 65/70] Clean-up hocon --- neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon | 1 - 1 file changed, 1 deletion(-) diff --git a/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon b/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon index 67bdca108..567230df8 100644 --- a/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon +++ b/neuro_san/registries/music_nerd_pro_llm_bedrock_claude.hocon @@ -16,7 +16,6 @@ { "llm_config": { "model_name": "bedrock-us-claude-3-7-sonnet", - "region_name": "us-west-2" }, "tools": [ # These tool definitions do not have to be in any particular order From af25a2422e793afb33e3dfeb10cab13880dc072a Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 17:32:49 -0700 Subject: [PATCH 66/70] Added new script --- .github/workflows/smoke.yml | 63 ++++++------------- .../set_test_aws_cli_list_bedrock_models.sh | 15 +++++ 2 files changed, 34 insertions(+), 44 deletions(-) create mode 100755 build_scripts/set_test_aws_cli_list_bedrock_models.sh diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index d0cf6ca41..3227502e0 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -50,54 +50,29 @@ jobs: pip install -r requirements-build.txt pip install -r requirements.txt - # - name: Set Ollama model to be used for related test case - # shell: bash - # run: | - # build_scripts/set_ollama_model.sh - - # - name: Show installed packages - # run: pip freeze - - - name: Install AWS CLI prerequisites - run: | - apt-get update - apt-get install -y unzip zip - - # 1) Install AWS CLI v2 (which includes the Bedrock commands) - - name: Install AWS CLI v2 + - name: Set Ollama model to be used for related test case + shell: bash run: | - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscliv2.zip - unzip awscliv2.zip - ./aws/install - aws --version + build_scripts/set_ollama_model.sh + + - name: Show installed packages + run: pip freeze - # - name: Set up AWS config & credentials + # - name: Install AWS CLI prerequisites # run: | - # mkdir -p ~/.aws + # apt-get update + # apt-get install -y unzip zip - # # Print to verify the vars are in the environment - # echo "ENV CHECK: AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" - - # # Write config - # printf "[default]\nregion=%s\noutput=json\n" \ - # "$AWS_REGION" \ - # > ~/.aws/config - - # # Write credentials - # printf "[default]\naws_access_key_id = %s\naws_secret_access_key = %s\n" \ - # "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" \ - # > ~/.aws/credentials - - # chmod 700 ~/.aws - # chmod 600 ~/.aws/config - # chmod 600 ~/.aws/credentials - - # echo "--- ~/.aws/config ---" && cat ~/.aws/config - # echo "--- ~/.aws/credentials ---" && cat ~/.aws/credentials - - - - name: List Bedrock foundation models - run: aws bedrock list-foundation-models + # # 1) Install AWS CLI v2 (which includes the Bedrock commands) + # - name: Install AWS CLI v2 + # run: | + # curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscliv2.zip + # unzip awscliv2.zip + # ./aws/install + # aws --version + + # - name: List Bedrock foundation models + # run: aws bedrock list-foundation-models - name: Start server & run smoke tests & print server-log if any failure shell: bash diff --git a/build_scripts/set_test_aws_cli_list_bedrock_models.sh b/build_scripts/set_test_aws_cli_list_bedrock_models.sh new file mode 100755 index 000000000..e510355f1 --- /dev/null +++ b/build_scripts/set_test_aws_cli_list_bedrock_models.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Script set up needed to do a quick test connection using AWS CLI to list all Bedrocks on this credential. + +# Install AWS CLI prerequisites +apt-get install -y unzip zip + +# Install AWS CLI v2 (which includes the Bedrock commands) +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscliv2.zip +unzip awscliv2.zip +./aws/install +aws --version + +# List Bedrock foundation models for single profile only +aws bedrock list-foundation-models From 15c887f79038da48bbcf071b8efca2523b376434 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 17:34:48 -0700 Subject: [PATCH 67/70] Clean up --- .github/workflows/smoke.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 3227502e0..182fb89b1 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -58,21 +58,8 @@ jobs: - name: Show installed packages run: pip freeze - # - name: Install AWS CLI prerequisites - # run: | - # apt-get update - # apt-get install -y unzip zip - - # # 1) Install AWS CLI v2 (which includes the Bedrock commands) - # - name: Install AWS CLI v2 - # run: | - # curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscliv2.zip - # unzip awscliv2.zip - # ./aws/install - # aws --version - - # - name: List Bedrock foundation models - # run: aws bedrock list-foundation-models + - name: Install and check access to Bedrock models + run: set_test_aws_cli_list_bedrock_models.sh - name: Start server & run smoke tests & print server-log if any failure shell: bash From 4477b6bd2c03bdbf670ea9c4fac9686fdceccfff Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 17:45:52 -0700 Subject: [PATCH 68/70] Tweak run shell file --- .github/workflows/smoke.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 182fb89b1..54a34baf4 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -59,7 +59,9 @@ jobs: run: pip freeze - name: Install and check access to Bedrock models - run: set_test_aws_cli_list_bedrock_models.sh + shell: bash + run: | + set_test_aws_cli_list_bedrock_models.sh - name: Start server & run smoke tests & print server-log if any failure shell: bash From abe8b46906d67d96f9be5b5a562f586b271c5cfa Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 17:57:43 -0700 Subject: [PATCH 69/70] Tweaked and clean-up --- .github/workflows/smoke.yml | 10 +++++----- ....sh => check_list_bedrock_models_profile_single.sh} | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename build_scripts/{set_test_aws_cli_list_bedrock_models.sh => check_list_bedrock_models_profile_single.sh} (100%) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 54a34baf4..80ee08779 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -54,14 +54,14 @@ jobs: shell: bash run: | build_scripts/set_ollama_model.sh - - - name: Show installed packages - run: pip freeze - - name: Install and check access to Bedrock models + - name: Check access to Bedrock models shell: bash run: | - set_test_aws_cli_list_bedrock_models.sh + build_scripts/check_list_bedrock_models_profile_single.sh + + - name: Show installed packages + run: pip freeze - name: Start server & run smoke tests & print server-log if any failure shell: bash diff --git a/build_scripts/set_test_aws_cli_list_bedrock_models.sh b/build_scripts/check_list_bedrock_models_profile_single.sh similarity index 100% rename from build_scripts/set_test_aws_cli_list_bedrock_models.sh rename to build_scripts/check_list_bedrock_models_profile_single.sh From c4d60fc952cf3a83680f3c6d2ef84e20628abe79 Mon Sep 17 00:00:00 2001 From: "vincent.nguyen" Date: Tue, 29 Jul 2025 18:03:39 -0700 Subject: [PATCH 70/70] re-enabled all test cases --- .../zzz_hocons/test_smoke_test_hocons.py | 158 +++++++++--------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py index 536af7a60..561091e69 100644 --- a/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py +++ b/tests/neuro_san/zzz_hocons/test_smoke_test_hocons.py @@ -29,64 +29,64 @@ class TestSmokeTestHocons(TestCase): # annotation below so the instance can find the hocon test cases listed. DYNAMIC = DynamicHoconUnitTests(__file__, path_to_basis="../../fixtures") - # @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ - # # These can be in any order. - # # Ideally more basic functionality will come first. - # # Barring that, try to stick to alphabetical order. - # "music_nerd_pro/combination_responses_with_history_direct.hocon", - - # # List more hocon files as they become available here. - # ])) - # @pytest.mark.timeout(30) # 30 seconds for this test - # @pytest.mark.smoke - # def test_hocon(self, test_name: str, test_hocon: str): - # """ - # Test method for a single parameterized test case specified by a hocon file. - # Arguments to this method are given by the iteration that happens as a result - # of the magic of the @parameterized.expand annotation above. - - # :param test_name: The name of a single test. - # :param test_hocon: The hocon file of a single data-driven test case. - # """ - # # Call the guts of the dynamic test driver. - # # This will expand the test_hocon file name from the expanded list to - # # include the file basis implied by the __file__ and path_to_basis above. - # self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) - - # @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ - # # These can be in any order. - # # Ideally more basic functionality will come first. - # # Barring that, try to stick to alphabetical order. - # # http has been covered by integration test - # # "music_nerd_pro/combination_responses_with_history_http.hocon", - # "music_nerd_pro/combination_responses_with_history_grpc.hocon", - - # # List more hocon files as they become available here. - # ])) - # @pytest.mark.timeout(30) # 30 seconds for this test - # @pytest.mark.smoke - # @pytest.mark.needs_server - # def test_hocon_with_server(self, test_name: str, test_hocon: str): - # """ - # Test method for a single parameterized test case specified by a hocon file. - # Arguments to this method are given by the iteration that happens as a result - # of the magic of the @parameterized.expand annotation above. - - # :param test_name: The name of a single test. - # :param test_hocon: The hocon file of a single data-driven test case. - # """ - # # Call the guts of the dynamic test driver. - # # This will expand the test_hocon file name from the expanded list to - # # include the file basis implied by the __file__ and path_to_basis above. - # self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) + @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ + # These can be in any order. + # Ideally more basic functionality will come first. + # Barring that, try to stick to alphabetical order. + "music_nerd_pro/combination_responses_with_history_direct.hocon", + + # List more hocon files as they become available here. + ])) + @pytest.mark.timeout(30) # 30 seconds for this test + @pytest.mark.smoke + def test_hocon(self, test_name: str, test_hocon: str): + """ + Test method for a single parameterized test case specified by a hocon file. + Arguments to this method are given by the iteration that happens as a result + of the magic of the @parameterized.expand annotation above. + + :param test_name: The name of a single test. + :param test_hocon: The hocon file of a single data-driven test case. + """ + # Call the guts of the dynamic test driver. + # This will expand the test_hocon file name from the expanded list to + # include the file basis implied by the __file__ and path_to_basis above. + self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) + + @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ + # These can be in any order. + # Ideally more basic functionality will come first. + # Barring that, try to stick to alphabetical order. + # http has been covered by integration test + # "music_nerd_pro/combination_responses_with_history_http.hocon", + "music_nerd_pro/combination_responses_with_history_grpc.hocon", + + # List more hocon files as they become available here. + ])) + @pytest.mark.timeout(30) # 30 seconds for this test + @pytest.mark.smoke + @pytest.mark.needs_server + def test_hocon_with_server(self, test_name: str, test_hocon: str): + """ + Test method for a single parameterized test case specified by a hocon file. + Arguments to this method are given by the iteration that happens as a result + of the magic of the @parameterized.expand annotation above. + + :param test_name: The name of a single test. + :param test_hocon: The hocon file of a single data-driven test case. + """ + # Call the guts of the dynamic test driver. + # This will expand the test_hocon file name from the expanded list to + # include the file basis implied by the __file__ and path_to_basis above. + self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ # These can be in any order. # Ideally more basic functionality will come first. # Barring that, try to stick to alphabetical order. - # "music_nerd_pro_llm_anthropic/combination_responses_with_history_direct.hocon", - # "music_nerd_pro_llm_gemini/combination_responses_with_history_direct.hocon", - # "music_nerd_pro_llm_azure/combination_responses_with_history_direct.hocon", + "music_nerd_pro_llm_anthropic/combination_responses_with_history_direct.hocon", + "music_nerd_pro_llm_gemini/combination_responses_with_history_direct.hocon", + "music_nerd_pro_llm_azure/combination_responses_with_history_direct.hocon", "music_nerd_pro_llm_bedrock_claude/combination_responses_with_history_direct.hocon", # List more hocon files as they become available here. @@ -112,29 +112,29 @@ def test_hocon_with_non_default_llm(self, test_name: str, test_hocon: str): # include the file basis implied by the __file__ and path_to_basis above. self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) - # @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ - # # These can be in any order. - # # Ideally more basic functionality will come first. - # # Barring that, try to stick to alphabetical order. - # "music_nerd_pro_llm_ollama/combination_responses_with_history_http.hocon", - - # # List more hocon files as they become available here. - # ])) - # @pytest.mark.timeout(12 * 60) # in mins for this test - # @pytest.mark.smoke - # @pytest.mark.needs_server - # @pytest.mark.non_default_llm_provider - # @pytest.mark.ollama - # def test_hocon_with_server_non_default_llm(self, test_name: str, test_hocon: str): - # """ - # Test method for a single parameterized test case specified by a hocon file. - # Arguments to this method are given by the iteration that happens as a result - # of the magic of the @parameterized.expand annotation above. - - # :param test_name: The name of a single test. - # :param test_hocon: The hocon file of a single data-driven test case. - # """ - # # Call the guts of the dynamic test driver. - # # This will expand the test_hocon file name from the expanded list to - # # include the file basis implied by the __file__ and path_to_basis above. - # self.DYNAMIC.one_test_hocon(self, test_name, test_hocon) + @parameterized.expand(DynamicHoconUnitTests.from_hocon_list([ + # These can be in any order. + # Ideally more basic functionality will come first. + # Barring that, try to stick to alphabetical order. + "music_nerd_pro_llm_ollama/combination_responses_with_history_http.hocon", + + # List more hocon files as they become available here. + ])) + @pytest.mark.timeout(12 * 60) # in mins for this test + @pytest.mark.smoke + @pytest.mark.needs_server + @pytest.mark.non_default_llm_provider + @pytest.mark.ollama + def test_hocon_with_server_non_default_llm(self, test_name: str, test_hocon: str): + """ + Test method for a single parameterized test case specified by a hocon file. + Arguments to this method are given by the iteration that happens as a result + of the magic of the @parameterized.expand annotation above. + + :param test_name: The name of a single test. + :param test_hocon: The hocon file of a single data-driven test case. + """ + # Call the guts of the dynamic test driver. + # This will expand the test_hocon file name from the expanded list to + # include the file basis implied by the __file__ and path_to_basis above. + self.DYNAMIC.one_test_hocon(self, test_name, test_hocon)