From 0b331d8be553820fc43a22c3f5e26034ad915fae Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Thu, 2 Oct 2025 14:22:14 -0700 Subject: [PATCH 01/10] Use LlmClient for Azure --- .../standard_langchain_llm_client_factory.py | 52 +++++++++++++++++-- .../llms/standard_langchain_llm_factory.py | 50 ++++++++++++------ 2 files changed, 82 insertions(+), 20 deletions(-) diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py index 6e6264c37..ec0f52ccc 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py @@ -63,7 +63,7 @@ def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: install_if_missing="langchain-openai") # Our run-time model resource here is httpx client which we need to control directly: - openai_proxy = self.get_value_or_env(config, "openai_organization", "OPENAI_PROXY") + openai_proxy = self.get_value_or_env(config, "openai_proxy", "OPENAI_PROXY") request_timeout = config.get("request_timeout") http_client = AsyncClient(proxy=openai_proxy, timeout=request_timeout) @@ -80,8 +80,54 @@ def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: elif chat_class == "azure-openai": - # Not yet - llm_client = None + # pylint: disable=invalid-name + AsyncAzureOpenAI = resolver.resolve_class_in_module("AsyncAzureOpenAI", + module_name="openai", + install_if_missing="langchain-openai") + + # Our run-time model resource here is httpx client which we need to control directly: + openai_proxy = self.get_value_or_env(config, "openai_proxy", "OPENAI_PROXY") + request_timeout = config.get("request_timeout") + http_client = AsyncClient(proxy=openai_proxy, timeout=request_timeout) + + # Prepare some more complex args + openai_api_key: str = self.get_value_or_env(config, "openai_api_key", "AZURE_OPENAI_API_KEY") + if openai_api_key is None: + openai_api_key = self.get_value_or_env(config, "openai_api_key", "OPENAI_API_KEY") + + # From lanchain_openai.chat_models.azure.py + default_headers: Dict[str, str] = {} + default_headers = config.get("default_headers", default_headers) + default_headers.update({ + "User-Agent": "langchain-partner-python-azure-openai", + }) + + async_azure_client = AsyncAzureOpenAI( + azure_endpoint=self.get_value_or_env(config, "azure_endpoint", + "AZURE_OPENAI_ENDPOINT"), + deployment_name=self.get_value_or_env(config, "deployment_name", + "AZURE_OPENAI_DEPLOYMENT_NAME"), + api_version=self.get_value_or_env(config, "openai_api_version", + "OPENAI_API_VERSION"), + api_key=openai_api_key, + + # AD here means "ActiveDirectory" + azure_ad_token=self.get_value_or_env(config, "azure_ad_token", + "AZURE_OPENAI_AD_TOKEN"), + # azure_ad_token_provider is a complex object, and we can't set that through config + + organization=self.get_value_or_env(config, "openai_organization", "OPENAI_ORG_ID"), + # project - not set in langchain_openai + # webhook_secret - not set in langchain_openai + base_url=self.get_value_or_env(config, "openai_api_base", "OPENAI_API_BASE"), + timeout=request_timeout, + max_retries=config.get("max_retries"), + default_headers=default_headers, + # default_query - don't understand enough to set, but set in langchain_openai + http_client=http_client + ) + + llm_client = HttpxLangChainLlmClient(http_client, async_azure_client) elif chat_class == "anthropic": diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py index c1f747590..41055e6ec 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py @@ -178,27 +178,42 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], "include_usage": True } } - openai_api_key: str = self.get_value_or_env(config, "openai_api_key", "AZURE_OPENAI_API_KEY") - if openai_api_key is None: - openai_api_key = self.get_value_or_env(config, "openai_api_key", "OPENAI_API_KEY") # AzureChatOpenAI just happens to come with langchain_openai # pylint: disable=invalid-name AzureChatOpenAI = resolver.resolve_class_in_module("AzureChatOpenAI", module_name="langchain_openai.chat_models.azure", install_if_missing="langchain-openai") + + # See if there is an async_client to be had from the llm_client passed in + async_client: Any = None + if llm_client is not None: + async_openai_client = llm_client.get_client() + if async_openai_client is not None: + # Necessary reach-in. + async_client = async_openai_client.chat.completions + + # Prepare some more complex args + openai_api_key: str = self.get_value_or_env(config, "openai_api_key", "AZURE_OPENAI_API_KEY", async_client) + if openai_api_key is None: + openai_api_key = self.get_value_or_env(config, "openai_api_key", "OPENAI_API_KEY", async_client) + llm = AzureChatOpenAI( + async_client=async_client, model_name=model_name, temperature=config.get("temperature"), + + # This next group of params should always be None when we have async_client openai_api_key=openai_api_key, openai_api_base=self.get_value_or_env(config, "openai_api_base", - "OPENAI_API_BASE"), + "OPENAI_API_BASE", async_client), openai_organization=self.get_value_or_env(config, "openai_organization", - "OPENAI_ORG_ID"), - openai_proxy=self.get_value_or_env(config, "openai_organization", - "OPENAI_PROXY"), - request_timeout=config.get("request_timeout"), - max_retries=config.get("max_retries"), + "OPENAI_ORG_ID", async_client), + openai_proxy=self.get_value_or_env(config, "openai_proxy", + "OPENAI_PROXY", async_client), + request_timeout=self.get_value_or_env(config, "request_timeout", None, async_client), + max_retries=self.get_value_or_env(config, "max_retries", None, async_client), + presence_penalty=config.get("presence_penalty"), frequency_penalty=config.get("frequency_penalty"), seed=config.get("seed"), @@ -227,20 +242,21 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], # global verbose value) so that the warning is never triggered. verbose=False, - # Azure-specific + # Azure-specific group that should be None if we have an async_client azure_endpoint=self.get_value_or_env(config, "azure_endpoint", - "AZURE_OPENAI_ENDPOINT"), + "AZURE_OPENAI_ENDPOINT", async_client), deployment_name=self.get_value_or_env(config, "deployment_name", - "AZURE_OPENAI_DEPLOYMENT_NAME"), + "AZURE_OPENAI_DEPLOYMENT_NAME", async_client), openai_api_version=self.get_value_or_env(config, "openai_api_version", - "OPENAI_API_VERSION"), - + "OPENAI_API_VERSION", async_client), # AD here means "ActiveDirectory" azure_ad_token=self.get_value_or_env(config, "azure_ad_token", - "AZURE_OPENAI_AD_TOKEN"), - model_version=config.get("model_version"), + "AZURE_OPENAI_AD_TOKEN", async_client), openai_api_type=self.get_value_or_env(config, "openai_api_type", - "OPENAI_API_TYPE"), + "OPENAI_API_TYPE", async_client), + + model_version=config.get("model_version"), + # Needed for token counting model_kwargs=model_kwargs, ) From 941d91c11295813d0b607de1fe5564a8055feac4 Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Thu, 2 Oct 2025 16:54:28 -0700 Subject: [PATCH 02/10] Attempt to create LlmClient after ChatAnthropic has been created --- .../standard_langchain_llm_client_factory.py | 3 +++ .../llms/standard_langchain_llm_factory.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py index ec0f52ccc..51ce5e342 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py @@ -132,6 +132,9 @@ def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: elif chat_class == "anthropic": # Not yet + # Anthropic models only support _async_client() as a cached_property, + # not as a passed-in arg. In LlmFactory, we grab hold of the async client + # after we create the ChatAntropic BaseLanguageModel. That's all we can do. llm_client = None elif chat_class == "ollama": diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py index 41055e6ec..ae3d69f03 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py @@ -16,6 +16,7 @@ from leaf_common.config.resolver import Resolver +from neuro_san.internals.run_context.langchain.llms.httpx_langchain_llm_client import HttpxLangChainLlmClient from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient from neuro_san.internals.run_context.langchain.llms.langchain_llm_factory import LangChainLlmFactory from neuro_san.internals.run_context.langchain.llms.langchain_llm_resources import LangChainLlmResources @@ -267,6 +268,10 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], ChatAnthropic = resolver.resolve_class_in_module("ChatAnthropic", module_name="langchain_anthropic.chat_models", install_if_missing="langchain-anthropic") + + # ChatAnthropic currently only supports _async_client() as a cached_property, + # not as a constructor arg. + llm = ChatAnthropic( model_name=model_name, max_tokens=config.get("max_tokens"), # This is always for output @@ -280,9 +285,14 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], "ANTHROPIC_API_URL"), anthropic_api_key=self.get_value_or_env(config, "anthropic_api_key", "ANTHROPIC_API_KEY"), + default_headers=config.get("default_headers"), + betas=config.get("betas"), streaming=True, # streaming is always on. Without it token counting will not work. # Set stream_usage to True in order to get token counting chunks. stream_usage=True, + thinking=config.get("thinking"), + mcp_servers=config.get("mcp_servers"), + context_management=config.get("context_management"), # If omitted, this defaults to the global verbose value, # accessible via langchain_core.globals.get_verbose(): @@ -299,6 +309,12 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], # global verbose value) so that the warning is never triggered. verbose=False, ) + + # Create the llm_client after the fact, with reach-in + async_anthropic = llm._async_client + http_client = async_anthropic.http_client + llm_client = HttpxLangChainLlmClient(http_client, async_anthropic) + elif chat_class == "ollama": # Use lazy loading to prevent installing the world From 2327b1c54f246e689e9fdbab226851b922af52d6 Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Thu, 2 Oct 2025 17:09:03 -0700 Subject: [PATCH 03/10] Add comments about bedrock --- .../langchain/llms/standard_langchain_llm_client_factory.py | 4 ++++ .../langchain/llms/standard_langchain_llm_factory.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py index 51ce5e342..c1932db0e 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py @@ -154,6 +154,10 @@ def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: elif chat_class == "bedrock": + # Note: ChatBedrock only ever uses a synchronous boto3 client to access + # any llm and there are no aioboto3 hooks yet. Not the greatest choice + # for a performant asynchronous server. + # Not yet llm_client = None diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py index ae3d69f03..7014d0ccf 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py @@ -431,6 +431,10 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], ) elif chat_class == "bedrock": + # Note: ChatBedrock only ever uses a synchronous boto3 client to access + # any llm and there are no aioboto3 hooks yet. Not the greatest choice + # for a performant asynchronous server. + # Use lazy loading to prevent installing the world # pylint: disable=invalid-name ChatBedrock = resolver.resolve_class_in_module("ChatBedrock", From 437310893524ed0cb8b78bc9e8ac30abf0bf0175 Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Thu, 2 Oct 2025 17:14:11 -0700 Subject: [PATCH 04/10] Fix pylint --- .../langchain/llms/standard_langchain_llm_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py index 7014d0ccf..20f26a39a 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py @@ -311,7 +311,7 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], ) # Create the llm_client after the fact, with reach-in - async_anthropic = llm._async_client + async_anthropic = llm._async_client # pylint:disable=protected-access http_client = async_anthropic.http_client llm_client = HttpxLangChainLlmClient(http_client, async_anthropic) From 8bd59d475d689b70d7c3e4ae9ecd610c8356c8f0 Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Fri, 3 Oct 2025 09:10:22 -0700 Subject: [PATCH 05/10] Create a BedrockLangChainLlmClient to hold the boto3 clients to close --- .../llms/bedrock_langchain_llm_client.py | 49 +++++++++++++++++++ .../standard_langchain_llm_client_factory.py | 9 ++-- .../llms/standard_langchain_llm_factory.py | 5 ++ 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py diff --git a/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py new file mode 100644 index 000000000..935340bf3 --- /dev/null +++ b/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py @@ -0,0 +1,49 @@ + +# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. +# All Rights Reserved. +# Issued under the Academic Public License. +# +# You can be released from the terms, and requirements of the Academic Public +# License by purchasing a commercial license. +# Purchase of a commercial license is mandatory for any use of the +# neuro-san SDK Software in commercial settings. +# +# END COPYRIGHT +from typing import Any + +from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient + + +class BedrockLangChainLlmClient(LangChainLlmClient): + """ + LangChainLlmClient implementation for Bedrock + """ + + def __init__(self, boto_client: Any, bedrock_client: Any): + """ + Constructor + + :param boto_client: boto client + :param bedrock_client: bedrock client + """ + self.boto_client = boto_client + self.bedrock_client = bedrock_client + + def get_client(self) -> Any: + """ + Get the client used by the model + """ + return self.boto_client + + async def delete_resources(self): + """ + Release the run-time resources used by the model + """ + # Neither of these clients are async. + if self.boto_client is not None: + self.boto_client.close() + self.boto_client = None + + if self.bedrock_client is not None: + self.bedrock_client.close() + self.bedrock_client = None diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py index c1932db0e..fe3775491 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py @@ -132,9 +132,9 @@ def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: elif chat_class == "anthropic": # Not yet - # Anthropic models only support _async_client() as a cached_property, - # not as a passed-in arg. In LlmFactory, we grab hold of the async client - # after we create the ChatAntropic BaseLanguageModel. That's all we can do. + # Anthropic models only support _async_client() as a cached_property, not as a passed-in arg. + # In LlmFactory, we grab hold of the client after we create the BaseLanguageModel. + # That's all we can do. llm_client = None elif chat_class == "ollama": @@ -157,8 +157,9 @@ def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: # Note: ChatBedrock only ever uses a synchronous boto3 client to access # any llm and there are no aioboto3 hooks yet. Not the greatest choice # for a performant asynchronous server. + # In LlmFactory, we grab hold of the client after we create the BaseLanguageModel. + # That's all we can do. - # Not yet llm_client = None elif chat_class is None: diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py index 20f26a39a..36a0cf450 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py @@ -16,6 +16,7 @@ from leaf_common.config.resolver import Resolver +from neuro_san.internals.run_context.langchain.llms.bedrock_langchain_llm_client import BedrockLangChainLlmClient from neuro_san.internals.run_context.langchain.llms.httpx_langchain_llm_client import HttpxLangChainLlmClient from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient from neuro_san.internals.run_context.langchain.llms.langchain_llm_factory import LangChainLlmFactory @@ -479,6 +480,10 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], # global verbose value) so that the warning is never triggered. verbose=False, ) + + # Create the llm_client after the fact, with reach-in + llm_client = BedrockLangChainLlmClient(llm.client, llm.bedrock_client) + elif chat_class is None: raise ValueError(f"Class name {chat_class} for model_name {model_name} is unspecified.") else: From 50cd2c0cd17e6bfa16b94a99ddbbf50623303eb9 Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Fri, 3 Oct 2025 14:02:38 -0700 Subject: [PATCH 06/10] Switch to a model where LlmClient is really a policy object instead of a data object --- .../llms/anthropic_langchain_llm_client.py | 48 +++++ .../llms/azure_langchain_llm_client.py | 98 ++++++++++ .../llms/bedrock_langchain_llm_client.py | 45 ++--- .../langchain/llms/default_llm_factory.py | 66 +------ .../llms/httpx_langchain_llm_client.py | 54 ------ .../langchain/llms/langchain_llm_client.py | 73 +++++++- .../llms/langchain_llm_client_factory.py | 52 ------ .../langchain/llms/langchain_llm_factory.py | 16 +- .../llms/openai_langchain_llm_client.py | 111 ++++++++++++ .../standard_langchain_llm_client_factory.py | 170 ------------------ .../llms/standard_langchain_llm_factory.py | 43 ++--- 11 files changed, 368 insertions(+), 408 deletions(-) create mode 100644 neuro_san/internals/run_context/langchain/llms/anthropic_langchain_llm_client.py create mode 100644 neuro_san/internals/run_context/langchain/llms/azure_langchain_llm_client.py delete mode 100644 neuro_san/internals/run_context/langchain/llms/httpx_langchain_llm_client.py delete mode 100644 neuro_san/internals/run_context/langchain/llms/langchain_llm_client_factory.py create mode 100644 neuro_san/internals/run_context/langchain/llms/openai_langchain_llm_client.py delete mode 100644 neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py diff --git a/neuro_san/internals/run_context/langchain/llms/anthropic_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/anthropic_langchain_llm_client.py new file mode 100644 index 000000000..d5a57d81f --- /dev/null +++ b/neuro_san/internals/run_context/langchain/llms/anthropic_langchain_llm_client.py @@ -0,0 +1,48 @@ + +# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. +# All Rights Reserved. +# Issued under the Academic Public License. +# +# You can be released from the terms, and requirements of the Academic Public +# License by purchasing a commercial license. +# Purchase of a commercial license is mandatory for any use of the +# neuro-san SDK Software in commercial settings. +# +# END COPYRIGHT + +from typing import Any + +from contextlib import suppress + +from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient + + +class AnthropicLangChainLlmClient(LangChainLlmClient): + """ + Implementation of the LangChainLlmClient for Anthtropic chat models. + + Anthropic chat models do not allow for passing in an externally managed + async web client. + """ + + async def delete_resources(self): + """ + Release the run-time resources used by the model + """ + if self.llm is None: + return + + # Do the necessary reach-ins to successfully shut down the web client + + # This is really an anthropic.AsyncClient, but we don't really want to do the Resolver here. + # Note we don't want to do this in the constructor, as AnthropicChat lazily + # creates these as needed via a cached_property that needs to be done in its own time + # via Anthropic infrastructure. By the time we get here, it's already been created. + anthropic_async_client: Any = self.llm._async_client # pylint:disable=protected-access + + if anthropic_async_client is not None: + with suppress(Exception): + await anthropic_async_client.aclose() + + # Let's not do this again, shall we? + self.llm = None diff --git a/neuro_san/internals/run_context/langchain/llms/azure_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/azure_langchain_llm_client.py new file mode 100644 index 000000000..c8b06abec --- /dev/null +++ b/neuro_san/internals/run_context/langchain/llms/azure_langchain_llm_client.py @@ -0,0 +1,98 @@ + +# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. +# All Rights Reserved. +# Issued under the Academic Public License. +# +# You can be released from the terms, and requirements of the Academic Public +# License by purchasing a commercial license. +# Purchase of a commercial license is mandatory for any use of the +# neuro-san SDK Software in commercial settings. +# +# END COPYRIGHT +from typing import Any +from typing import Dict + +from leaf_common.config.resolver import Resolver + +from neuro_san.internals.run_context.langchain.llms.openai_langchain_llm_client import OpenAILangChainLlmClient + + +class AzureLangChainLlmClient(OpenAILangChainLlmClient): + """ + LangChainLlmClient implementation for OpenAI via Azure. + + OpenAI's BaseLanguageModel implementations do allow us to pass in a web client + as an argument, so this implementation takes advantage of the create_client() + method to do that. Worth noting that where many other implementations might care about + the llm reference, because of our create_client() implementation, we do not. + """ + + def create_client(self, config: Dict[str, Any]) -> Any: + """ + Creates the web client to used by a BaseLanguageModel to be + constructed in the future. Neuro SAN infrastructures prefers that this + be an asynchronous client, however we realize some BaseLanguageModels + do not support that (even though they should!). + + Implementations should retain any references to state that needs to be cleaned up + in the delete_resources() method. + + :param config: The fully specified llm config + :return: The web client that accesses the LLM. + By default this is None, as many BaseLanguageModels + do not allow a web client to be passed in as an arg. + """ + # OpenAI is the one chat class that we do not require any extra installs. + # This is what we want to work out of the box. + # Nevertheless, have it go through the same lazy-loading resolver rigamarole as the others. + + # Set up a resolver to use to resolve lazy imports of classes from + # langchain_* packages to prevent installing the world. + resolver = Resolver() + + # pylint: disable=invalid-name + AsyncAzureOpenAI = resolver.resolve_class_in_module("AsyncAzureOpenAI", + module_name="openai", + install_if_missing="langchain-openai") + + self.create_http_client(config) + + # Prepare some more complex args + openai_api_key: str = self.get_value_or_env(config, "openai_api_key", "AZURE_OPENAI_API_KEY") + if openai_api_key is None: + openai_api_key = self.get_value_or_env(config, "openai_api_key", "OPENAI_API_KEY") + + # From lanchain_openai.chat_models.azure.py + default_headers: Dict[str, str] = {} + default_headers = config.get("default_headers", default_headers) + default_headers.update({ + "User-Agent": "langchain-partner-python-azure-openai", + }) + + self.async_openai_client = AsyncAzureOpenAI( + azure_endpoint=self.get_value_or_env(config, "azure_endpoint", + "AZURE_OPENAI_ENDPOINT"), + deployment_name=self.get_value_or_env(config, "deployment_name", + "AZURE_OPENAI_DEPLOYMENT_NAME"), + api_version=self.get_value_or_env(config, "openai_api_version", + "OPENAI_API_VERSION"), + api_key=openai_api_key, + # AD here means "ActiveDirectory" + azure_ad_token=self.get_value_or_env(config, "azure_ad_token", + "AZURE_OPENAI_AD_TOKEN"), + # azure_ad_token_provider is a complex object, and we can't set that through config + + organization=self.get_value_or_env(config, "openai_organization", "OPENAI_ORG_ID"), + # project - not set in langchain_openai + # webhook_secret - not set in langchain_openai + base_url=self.get_value_or_env(config, "openai_api_base", "OPENAI_API_BASE"), + timeout=config.get("request_timeout"), + max_retries=config.get("max_retries"), + default_headers=default_headers, + # default_query - don't understand enough to set, but set in langchain_openai + http_client=self.http_client + ) + + # We retain the async_openai_client reference, but we hand back this reach-in + # to pass to the BaseLanguageModel constructor. + return self.async_openai_client.chat.completions diff --git a/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py index 935340bf3..805e3afbd 100644 --- a/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py +++ b/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py @@ -9,41 +9,34 @@ # neuro-san SDK Software in commercial settings. # # END COPYRIGHT -from typing import Any from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient class BedrockLangChainLlmClient(LangChainLlmClient): """ - LangChainLlmClient implementation for Bedrock - """ - - def __init__(self, boto_client: Any, bedrock_client: Any): - """ - Constructor + LangChainLlmClient implementation for Bedrock. - :param boto_client: boto client - :param bedrock_client: bedrock client - """ - self.boto_client = boto_client - self.bedrock_client = bedrock_client - - def get_client(self) -> Any: - """ - Get the client used by the model - """ - return self.boto_client + Bedrock does not allow for passing in async web clients. + As a matter of fact, all of its clients are synchronous, + which is not the best for an async service. + """ async def delete_resources(self): """ Release the run-time resources used by the model """ - # Neither of these clients are async. - if self.boto_client is not None: - self.boto_client.close() - self.boto_client = None - - if self.bedrock_client is not None: - self.bedrock_client.close() - self.bedrock_client = None + if self.llm is None: + return + + # Do the necessary reach-ins to successfully shut down the web client + if self.llm.client is not None: + # This is a boto3 client + self.llm.client.close() + + if self.llm.bedrock_client is not None: + # This is a boto3 client + self.llm.bedrock_client.close() + + # Let's not do this again, shall we? + self.llm = None diff --git a/neuro_san/internals/run_context/langchain/llms/default_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/default_llm_factory.py index 395a28b9c..d88b5f577 100644 --- a/neuro_san/internals/run_context/langchain/llms/default_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/default_llm_factory.py @@ -26,13 +26,9 @@ from leaf_common.parsers.dictionary_extractor import DictionaryExtractor from neuro_san.internals.interfaces.context_type_llm_factory import ContextTypeLlmFactory -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client_factory import LangChainLlmClientFactory from neuro_san.internals.run_context.langchain.llms.langchain_llm_factory import LangChainLlmFactory from neuro_san.internals.run_context.langchain.llms.langchain_llm_resources import LangChainLlmResources from neuro_san.internals.run_context.langchain.llms.llm_info_restorer import LlmInfoRestorer -from neuro_san.internals.run_context.langchain.llms.standard_langchain_llm_client_factory \ - import StandardLangChainLlmClientFactory from neuro_san.internals.run_context.langchain.llms.standard_langchain_llm_factory import StandardLangChainLlmFactory from neuro_san.internals.run_context.langchain.util.api_key_error_check import ApiKeyErrorCheck from neuro_san.internals.run_context.langchain.util.argument_validator import ArgumentValidator @@ -86,9 +82,6 @@ def __init__(self, config: Dict[str, Any] = None): self.llm_factories: List[LangChainLlmFactory] = [ StandardLangChainLlmFactory() ] - self.llm_client_factories: List[LangChainLlmClientFactory] = [ - StandardLangChainLlmClientFactory() - ] # Get user LLM info file path with the following priority: # 1. "agent_llm_info_file" from agent network hocon @@ -178,8 +171,7 @@ def create_llm( unknown to this method. """ full_config: Dict[str, Any] = self.create_full_llm_config(config) - llm_client: LangChainLlmClient = self.create_llm_client(full_config) - llm_resources: LangChainLlmResources = self.create_llm_resources_with_client(full_config, llm_client) + llm_resources: LangChainLlmResources = self.create_llm_resources(full_config) return llm_resources def create_full_llm_config(self, config: Dict[str, Any]) -> Dict[str, Any]: @@ -279,52 +271,6 @@ def get_chat_class_args(self, chat_class_name: str, use_model_name: str = None) return args - def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: - """ - Create a LangChainLlmClient from the fully-specified llm config either from - StandardLlmClientFactory or a user-defined LlmClientFactory. - :param config: The fully specified llm config which is a product of - _create_full_llm_config() above. - :return: A LangChainLlmClient instance to use when making llm requests. - This can be None if client connection management is not required. - Can raise a ValueError if the config's class or model_name value is - unknown to this method. - """ - llm_client: LangChainLlmClient = None - - # Loop through the loaded factories in order until we can find one - # that can create the llm. - found_exception: Exception = None - for llm_client_factory in self.llm_client_factories: - try: - llm_client = llm_client_factory.create_llm_client(config) - if llm_client is not None and isinstance(llm_client, LangChainLlmClient): - # We found what we were looking for - found_exception = None - break - - # Catch some common wrong or missing API key errors in a single place - # with some verbose error messaging. - except API_KEY_ERRORS as exception: - # Will re-raise but with the right exception text it will - # also provide some more helpful failure text. - message: str = ApiKeyErrorCheck.check_for_api_key_exception(exception) - if message is not None: - raise ValueError(message) from exception - found_exception = exception - - except ValueError as exception: - # Let the next model have a crack - found_exception = exception - - # DEF - Might eventually want to resolve a specific class like the end of - # create_llm_resources_with_client() does. - - if found_exception is not None: - raise found_exception - - return llm_client - def create_base_chat_model(self, config: Dict[str, Any]) -> BaseLanguageModel: """ Create a BaseLanguageModel from the fully-specified llm config. @@ -336,18 +282,12 @@ def create_base_chat_model(self, config: Dict[str, Any]) -> BaseLanguageModel: """ raise NotImplementedError - def create_llm_resources_with_client(self, config: Dict[str, Any], - llm_client: LangChainLlmClient = None) -> LangChainLlmResources: + def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: """ Create a BaseLanguageModel from the fully-specified llm config either from standard LLM factory, user-defined LLM factory, or user-specified langchain model class. :param config: The fully specified llm config which is a product of _create_full_llm_config() above. - :param llm_client: A LangChainLlmClient instance, which by default is None, - implying that create_base_chat_model() needs to create its own client. - Note, however that a None value can lead to connection leaks and requests - that continue to run after the request connection is dropped in a server - environment. :return: A LangChainLlmResources instance containing a BaseLanguageModel (can be Chat or LLM) and all related resources necessary for managing the model run-time lifecycle. @@ -361,7 +301,7 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], found_exception: Exception = None for llm_factory in self.llm_factories: try: - llm_resources = llm_factory.create_llm_resources_with_client(config, llm_client) + llm_resources = llm_factory.create_llm_resources(config) if llm_resources is not None and isinstance(llm_resources, LangChainLlmResources): # We found what we were looking for found_exception = None diff --git a/neuro_san/internals/run_context/langchain/llms/httpx_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/httpx_langchain_llm_client.py deleted file mode 100644 index 53bcd34fa..000000000 --- a/neuro_san/internals/run_context/langchain/llms/httpx_langchain_llm_client.py +++ /dev/null @@ -1,54 +0,0 @@ - -# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. -# All Rights Reserved. -# Issued under the Academic Public License. -# -# You can be released from the terms, and requirements of the Academic Public -# License by purchasing a commercial license. -# Purchase of a commercial license is mandatory for any use of the -# neuro-san SDK Software in commercial settings. -# -# END COPYRIGHT - -from typing import Any - -from contextlib import suppress -from httpx import AsyncClient - -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient - - -class HttpxLangChainLlmClient(LangChainLlmClient): - """ - Implementation of the LangChainLlmClient for httpx clients. - """ - - def __init__(self, http_client: AsyncClient, async_llm_client: Any = None): - """ - Constructor. - - :param http_client: httpx.AsyncClient used for model connections to LLM host. - :param async_llm_client: optional async_llm_client used for model connections to LLM host. - When used, this is most often ChatModel-specific. - """ - self.http_client: AsyncClient = http_client - self.async_llm_client: Any = async_llm_client - - def get_client(self) -> Any: - """ - Get the async llm client used by the model - """ - return self.async_llm_client - - async def delete_resources(self): - """ - Release the run-time resources used by the model - """ - self.async_llm_client = None - - if self.http_client is None: - return - - with suppress(Exception): - await self.http_client.aclose() - self.http_client = None diff --git a/neuro_san/internals/run_context/langchain/llms/langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/langchain_llm_client.py index 616afedf9..0fb4eced2 100644 --- a/neuro_san/internals/run_context/langchain/llms/langchain_llm_client.py +++ b/neuro_san/internals/run_context/langchain/llms/langchain_llm_client.py @@ -10,21 +10,84 @@ # # END COPYRIGHT from typing import Any +from typing import Dict + +import os + +from langchain.llms.base import BaseLanguageModel class LangChainLlmClient: """ - Interface for representing a client that connects to a LangChain model + Policy interface to manage the lifecycles of web clients that talk to LLM services. + + There are really two styles of implementation encompassed by this one interface. + + 1) When BaseLanguageModels can have web clients passed into their constructor, + implementations should use the create_client() method to retain any references necessary + to help them clean up nicely in the delete_resources() method. + + 2) When BaseLanguageModels cannot have web clients passed into their constructor, + implementations should pass the already created llm into their implementation's + constructor. Later delete_resources() implementations will need to do a reach-in + to the llm instance to clean up any references related to the web client. """ - def get_client(self) -> Any: + def __init__(self, llm: BaseLanguageModel = None): """ - Get the client used by the model + Constructor. + + :param llm: BaseLanguageModel """ - raise NotImplementedError + self.llm: BaseLanguageModel = llm + + # pylint: disable=useless-return + def create_client(self, config: Dict[str, Any]) -> Any: + """ + Creates the web client to used by a BaseLanguageModel to be + constructed in the future. Neuro SAN infrastructures prefers that this + be an asynchronous client, however we realize some BaseLanguageModels + do not support that (even though they should!). + + Implementations should retain any references to state that needs to be cleaned up + in the delete_resources() method. + + :param config: The fully specified llm config + :return: The web client that accesses the LLM. + By default this is None, as many BaseLanguageModels + do not allow a web client to be passed in as an arg. + """ + _ = config + return None async def delete_resources(self): """ - Release the run-time resources used by the model + Release the run-time resources used by the instance. + + Unfortunately for many BaseLanguageModels, this tends to involve + a reach-in to its private internals in order to shutting down + any web client references in there. """ raise NotImplementedError + + def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str, + none_obj: Any = None) -> Any: + """ + :param config: The config dictionary to search + :param key: The key for the config to look for + :param env_key: The os.environ key whose value should be gotten if either + the key does not exist or the value for the key is None + :param none_obj: An optional object instance to test. + If present this method will return None. + """ + if none_obj is not None: + return None + + value = None + if config is not None: + value = config.get(key) + + if value is None and env_key is not None: + value = os.getenv(env_key) + + return value diff --git a/neuro_san/internals/run_context/langchain/llms/langchain_llm_client_factory.py b/neuro_san/internals/run_context/langchain/llms/langchain_llm_client_factory.py deleted file mode 100644 index 1c7bf8617..000000000 --- a/neuro_san/internals/run_context/langchain/llms/langchain_llm_client_factory.py +++ /dev/null @@ -1,52 +0,0 @@ - -# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. -# All Rights Reserved. -# Issued under the Academic Public License. -# -# You can be released from the terms, and requirements of the Academic Public -# License by purchasing a commercial license. -# Purchase of a commercial license is mandatory for any use of the -# neuro-san SDK Software in commercial settings. -# -# END COPYRIGHT - -from typing import Any -from typing import Dict - -import os - -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient - - -class LangChainLlmClientFactory: - """ - Interface for Factory classes creating LLM client connections for LangChain. - """ - - def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: - """ - Create a LangChainLlmClient instance from the fully-specified llm config. - :param config: The fully specified llm config - :return: A LangChainLlmClient instance containing run-time resources necessary - for model usage by the service. - - Can raise a ValueError if the config's class or model_name value is - unknown to this method. - """ - raise NotImplementedError - - def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str) -> Any: - """ - :param config: The config dictionary to search - :param key: The key for the config to look for - :param env_key: The os.environ key whose value should be gotten if either - the key does not exist or the value for the key is None - """ - value = None - if config is not None: - value = config.get(key) - - if value is None: - value = os.getenv(env_key) - - return value diff --git a/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py index cd6d3d3f3..5f8766b00 100644 --- a/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py @@ -17,7 +17,6 @@ from langchain_core.language_models.base import BaseLanguageModel -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient from neuro_san.internals.run_context.langchain.llms.langchain_llm_resources import LangChainLlmResources @@ -53,7 +52,7 @@ def create_base_chat_model(self, config: Dict[str, Any]) -> BaseLanguageModel: Create a LangChainLlmResources instance from the fully-specified llm config. This method is provided for backwards compatibility. - Prefer create_llm_resources_with_client() instead, + Prefer create_llm_resources() instead, as this allows server infrastructure to better account for outstanding connections to LLM providers when connections drop. @@ -65,20 +64,15 @@ def create_base_chat_model(self, config: Dict[str, Any]) -> BaseLanguageModel: """ raise NotImplementedError - def create_llm_resources_with_client(self, config: Dict[str, Any], - llm_client: LangChainLlmClient = None) -> LangChainLlmResources: + def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: """ Create a LangChainLlmResources instance from the fully-specified llm config. :param config: The fully specified llm config which is a product of _create_full_llm_config() above. - :param llm_client: A LangChainLlmClient instance, which by default is None, - implying that create_base_chat_model() needs to create its own client. - Note, however that a None value can lead to connection leaks and requests - that continue to run after the request connection is dropped in a server - environment. :return: A LangChainLlmResources instance containing - a BaseLanguageModel (can be Chat or LLM) and all related resources + a BaseLanguageModel (can be Chat or LLM) and an LangChainLlmClient + policy object that contains all related resources necessary for managing the model run-time lifecycle. Can raise a ValueError if the config's class or model_name value is unknown to this method. @@ -95,7 +89,7 @@ def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str, :param llm_client: An optional client instance. If present this method will return None. - Most BaseLanguageModels will take some kind of pre-made + Some BaseLanguageModels will take some kind of pre-made client as part of their constructor args, but they will also take enough args to constructor a client for themselves under the hood when explicitly not given that client. diff --git a/neuro_san/internals/run_context/langchain/llms/openai_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/openai_langchain_llm_client.py new file mode 100644 index 000000000..aefc4f9c5 --- /dev/null +++ b/neuro_san/internals/run_context/langchain/llms/openai_langchain_llm_client.py @@ -0,0 +1,111 @@ + +# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. +# All Rights Reserved. +# Issued under the Academic Public License. +# +# You can be released from the terms, and requirements of the Academic Public +# License by purchasing a commercial license. +# Purchase of a commercial license is mandatory for any use of the +# neuro-san SDK Software in commercial settings. +# +# END COPYRIGHT +from typing import Any +from typing import Dict + +from contextlib import suppress +from httpx import AsyncClient + +from langchain_core.language_models.base import BaseLanguageModel + +from leaf_common.config.resolver import Resolver + +from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient + + +class OpenAILangChainLlmClient(LangChainLlmClient): + """ + LangChainLlmClient implementation for OpenAI. + + OpenAI's BaseLanguageModel implementations do allow us to pass in a web client + as an argument, so this implementation takes advantage of the create_client() + method to do that. Worth noting that where many other implementations might care about + the llm reference, because of our create_client() implementation, we do not. + """ + + def __init__(self, llm: BaseLanguageModel = None): + """ + Constructor. + """ + super().__init__() + + self.http_client: AsyncClient = None + + # Not doing lazy type resolution here just for type hints. + # Save that for create_client(), where it's meatier. + self.async_openai_client: Any = None + + def create_client(self, config: Dict[str, Any]) -> Any: + """ + Creates the web client to used by a BaseLanguageModel to be + constructed in the future. Neuro SAN infrastructures prefers that this + be an asynchronous client, however we realize some BaseLanguageModels + do not support that (even though they should!). + + Implementations should retain any references to state that needs to be cleaned up + in the delete_resources() method. + + :param config: The fully specified llm config + :return: The web client that accesses the LLM. + By default this is None, as many BaseLanguageModels + do not allow a web client to be passed in as an arg. + """ + # OpenAI is the one chat class that we do not require any extra installs. + # This is what we want to work out of the box. + # Nevertheless, have it go through the same lazy-loading resolver rigamarole as the others. + + # Set up a resolver to use to resolve lazy imports of classes from + # langchain_* packages to prevent installing the world. + resolver = Resolver() + + # pylint: disable=invalid-name + AsyncOpenAI = resolver.resolve_class_in_module("AsyncOpenAI", + module_name="openai", + install_if_missing="langchain-openai") + + self.create_http_client(config) + + self.async_openai_client = AsyncOpenAI( + api_key=self.get_value_or_env(config, "openai_api_key", "OPENAI_API_KEY"), + base_url=self.get_value_or_env(config, "openai_api_base", "OPENAI_API_BASE"), + organization=self.get_value_or_env(config, "openai_organization", "OPENAI_ORG_ID"), + timeout=config.get("request_timeout"), + max_retries=config.get("max_retries"), + http_client=self.http_client + ) + + # We retain the async_openai_client reference, but we hand back this reach-in + # to pass to the BaseLanguageModel constructor. + return self.async_openai_client.chat.completions + + def create_http_client(self, config: Dict[str, Any]): + """ + Creates the http client from the given config. + + :param config: The fully specified llm config + """ + # Our run-time model resource here is httpx client which we need to control directly: + openai_proxy: str = self.get_value_or_env(config, "openai_proxy", "OPENAI_PROXY") + request_timeout: int = config.get("request_timeout") + self.http_client = AsyncClient(proxy=openai_proxy, timeout=request_timeout) + + async def delete_resources(self): + """ + Release the run-time resources used by the instance. + """ + self.async_openai_client = None + + if self.http_client is not None: + with suppress(Exception): + await self.http_client.aclose() + + self.http_client = None diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py deleted file mode 100644 index fe3775491..000000000 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_client_factory.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. -# All Rights Reserved. -# Issued under the Academic Public License. -# -# You can be released from the terms, and requirements of the Academic Public -# License by purchasing a commercial license. -# Purchase of a commercial license is mandatory for any use of the -# neuro-san SDK Software in commercial settings. -# -# END COPYRIGHT - -from typing import Any -from typing import Dict - -from httpx import AsyncClient - -from leaf_common.config.resolver import Resolver - -from neuro_san.internals.run_context.langchain.llms.httpx_langchain_llm_client import HttpxLangChainLlmClient -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client_factory import LangChainLlmClientFactory - - -class StandardLangChainLlmClientFactory(LangChainLlmClientFactory): - """ - Factory class for creating LangChainLlmClient instances for Chat and LLM operations - """ - - def create_llm_client(self, config: Dict[str, Any]) -> LangChainLlmClient: - """ - Create an LangChainLlmClient instance from the fully-specified llm config. - :param config: The fully specified llm config - :return: A LangChainLlmClient instance. - Can be None if the llm class in the config does not need a client. - Can raise a ValueError if the config's class or model_name value is - unknown to this method. - """ - # pylint: disable=too-many-locals - # Construct the LLM - llm_client: LangChainLlmClient = None - chat_class: str = config.get("class") - if chat_class is not None: - chat_class = chat_class.lower() - - # Check for key "model_name", "model", and "model_id" to use as model name - # If the config is from default_llm_info, this is always "model_name" - # but with user-specified config, it is possible to have the other keys will be specifed instead. - model_name: str = config.get("model_name") or config.get("model") or config.get("model_id") - - # Set up a resolver to use to resolve lazy imports of classes from - # langchain_* packages to prevent installing the world. - resolver = Resolver() - - if chat_class == "openai": - - # OpenAI is the one chat class that we do not require any extra installs. - # This is what we want to work out of the box. - # Nevertheless, have it go through the same lazy-loading resolver rigamarole as the others. - - # pylint: disable=invalid-name - AsyncOpenAI = resolver.resolve_class_in_module("AsyncOpenAI", - module_name="openai", - install_if_missing="langchain-openai") - - # Our run-time model resource here is httpx client which we need to control directly: - openai_proxy = self.get_value_or_env(config, "openai_proxy", "OPENAI_PROXY") - request_timeout = config.get("request_timeout") - http_client = AsyncClient(proxy=openai_proxy, timeout=request_timeout) - - async_openai_client = AsyncOpenAI( - api_key=self.get_value_or_env(config, "openai_api_key", "OPENAI_API_KEY"), - base_url=self.get_value_or_env(config, "openai_api_base", "OPENAI_API_BASE"), - organization=self.get_value_or_env(config, "openai_organization", "OPENAI_ORG_ID"), - timeout=request_timeout, - max_retries=config.get("max_retries"), - http_client=http_client - ) - - llm_client = HttpxLangChainLlmClient(http_client, async_openai_client) - - elif chat_class == "azure-openai": - - # pylint: disable=invalid-name - AsyncAzureOpenAI = resolver.resolve_class_in_module("AsyncAzureOpenAI", - module_name="openai", - install_if_missing="langchain-openai") - - # Our run-time model resource here is httpx client which we need to control directly: - openai_proxy = self.get_value_or_env(config, "openai_proxy", "OPENAI_PROXY") - request_timeout = config.get("request_timeout") - http_client = AsyncClient(proxy=openai_proxy, timeout=request_timeout) - - # Prepare some more complex args - openai_api_key: str = self.get_value_or_env(config, "openai_api_key", "AZURE_OPENAI_API_KEY") - if openai_api_key is None: - openai_api_key = self.get_value_or_env(config, "openai_api_key", "OPENAI_API_KEY") - - # From lanchain_openai.chat_models.azure.py - default_headers: Dict[str, str] = {} - default_headers = config.get("default_headers", default_headers) - default_headers.update({ - "User-Agent": "langchain-partner-python-azure-openai", - }) - - async_azure_client = AsyncAzureOpenAI( - azure_endpoint=self.get_value_or_env(config, "azure_endpoint", - "AZURE_OPENAI_ENDPOINT"), - deployment_name=self.get_value_or_env(config, "deployment_name", - "AZURE_OPENAI_DEPLOYMENT_NAME"), - api_version=self.get_value_or_env(config, "openai_api_version", - "OPENAI_API_VERSION"), - api_key=openai_api_key, - - # AD here means "ActiveDirectory" - azure_ad_token=self.get_value_or_env(config, "azure_ad_token", - "AZURE_OPENAI_AD_TOKEN"), - # azure_ad_token_provider is a complex object, and we can't set that through config - - organization=self.get_value_or_env(config, "openai_organization", "OPENAI_ORG_ID"), - # project - not set in langchain_openai - # webhook_secret - not set in langchain_openai - base_url=self.get_value_or_env(config, "openai_api_base", "OPENAI_API_BASE"), - timeout=request_timeout, - max_retries=config.get("max_retries"), - default_headers=default_headers, - # default_query - don't understand enough to set, but set in langchain_openai - http_client=http_client - ) - - llm_client = HttpxLangChainLlmClient(http_client, async_azure_client) - - elif chat_class == "anthropic": - - # Not yet - # Anthropic models only support _async_client() as a cached_property, not as a passed-in arg. - # In LlmFactory, we grab hold of the client after we create the BaseLanguageModel. - # That's all we can do. - llm_client = None - - elif chat_class == "ollama": - - # Never. Ollama models are local - llm_client = None - - elif chat_class == "nvidia": - - # Not yet - llm_client = None - - elif chat_class == "gemini": - - # Not yet - llm_client = None - - elif chat_class == "bedrock": - - # Note: ChatBedrock only ever uses a synchronous boto3 client to access - # any llm and there are no aioboto3 hooks yet. Not the greatest choice - # for a performant asynchronous server. - # In LlmFactory, we grab hold of the client after we create the BaseLanguageModel. - # That's all we can do. - - llm_client = None - - elif chat_class is None: - raise ValueError(f"Class name {chat_class} for model_name {model_name} is unspecified.") - else: - raise ValueError(f"Class {chat_class} for model_name {model_name} is unrecognized.") - - return llm_client diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py index 36a0cf450..7357bb414 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py @@ -16,8 +16,10 @@ from leaf_common.config.resolver import Resolver +from neuro_san.internals.run_context.langchain.llms.anthropic_langchain_llm_client import AnthropicLangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.azure_langchain_llm_client import AzureLangChainLlmClient from neuro_san.internals.run_context.langchain.llms.bedrock_langchain_llm_client import BedrockLangChainLlmClient -from neuro_san.internals.run_context.langchain.llms.httpx_langchain_llm_client import HttpxLangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.openai_langchain_llm_client import OpenAILangChainLlmClient from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient from neuro_san.internals.run_context.langchain.llms.langchain_llm_factory import LangChainLlmFactory from neuro_san.internals.run_context.langchain.llms.langchain_llm_resources import LangChainLlmResources @@ -55,7 +57,7 @@ def create_base_chat_model(self, config: Dict[str, Any]) -> BaseLanguageModel: Create a BaseLanguageModel from the fully-specified llm config. This method is provided for backwards compatibility. - Prefer create_llm_resources_with_client() instead, + Prefer create_llm_resources() instead, as this allows server infrastructure to better account for outstanding connections to LLM providers when connections drop. @@ -68,17 +70,11 @@ def create_base_chat_model(self, config: Dict[str, Any]) -> BaseLanguageModel: raise NotImplementedError # pylint: disable=too-many-branches - def create_llm_resources_with_client(self, config: Dict[str, Any], - llm_client: LangChainLlmClient = None) -> LangChainLlmResources: + def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: """ Create a BaseLanguageModel from the fully-specified llm config. :param config: The fully specified llm config which is a product of _create_full_llm_config() above. - :param llm_client: A LangChainLlmClient instance, which by default is None, - implying that create_base_chat_model() needs to create its own client. - Note, however that a None value can lead to connection leaks and requests - that continue to run after the request connection is dropped in a server - environment. :return: A LangChainLlmResources instance containing a BaseLanguageModel (can be Chat or LLM) and all related resources necessary for managing the model run-time lifecycle. @@ -88,6 +84,8 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], # pylint: disable=too-many-locals # Construct the LLM llm: BaseLanguageModel = None + llm_client: LangChainLlmClient = None + chat_class: str = config.get("class") if chat_class is not None: chat_class = chat_class.lower() @@ -112,13 +110,9 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], module_name="langchain_openai.chat_models.base", install_if_missing="langchain-openai") - # See if there is an async_client to be had from the llm_client passed in - async_client: Any = None - if llm_client is not None: - async_openai_client = llm_client.get_client() - if async_openai_client is not None: - # Necessary reach-in. - async_client = async_openai_client.chat.completions + # Create the policy object that allows us to manage the model run-time lifecycle + llm_client = OpenAILangChainLlmClient() + async_client: Any = llm_client.create_client(config) # Now construct LLM chat model we will be using: llm = ChatOpenAI( @@ -187,13 +181,9 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], module_name="langchain_openai.chat_models.azure", install_if_missing="langchain-openai") - # See if there is an async_client to be had from the llm_client passed in - async_client: Any = None - if llm_client is not None: - async_openai_client = llm_client.get_client() - if async_openai_client is not None: - # Necessary reach-in. - async_client = async_openai_client.chat.completions + # Create the policy object that allows us to manage the model run-time lifecycle + llm_client = AzureLangChainLlmClient() + async_client: Any = llm_client.create_client(config) # Prepare some more complex args openai_api_key: str = self.get_value_or_env(config, "openai_api_key", "AZURE_OPENAI_API_KEY", async_client) @@ -262,6 +252,7 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], # Needed for token counting model_kwargs=model_kwargs, ) + elif chat_class == "anthropic": # Use lazy loading to prevent installing the world @@ -312,9 +303,7 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], ) # Create the llm_client after the fact, with reach-in - async_anthropic = llm._async_client # pylint:disable=protected-access - http_client = async_anthropic.http_client - llm_client = HttpxLangChainLlmClient(http_client, async_anthropic) + llm_client = AnthropicLangChainLlmClient(llm) elif chat_class == "ollama": @@ -482,7 +471,7 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], ) # Create the llm_client after the fact, with reach-in - llm_client = BedrockLangChainLlmClient(llm.client, llm.bedrock_client) + llm_client = BedrockLangChainLlmClient(llm) elif chat_class is None: raise ValueError(f"Class name {chat_class} for model_name {model_name} is unspecified.") From 420d0814f2c4d998660f9e21fad9d17973013c4c Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Fri, 3 Oct 2025 14:16:35 -0700 Subject: [PATCH 07/10] Rename LangChainLlmClient -> ClientPolicy --- ...m_client.py => anthropic_client_policy.py} | 6 ++-- ...n_llm_client.py => azure_client_policy.py} | 6 ++-- ...llm_client.py => bedrock_client_policy.py} | 6 ++-- ...ngchain_llm_client.py => client_policy.py} | 2 +- .../langchain/llms/default_llm_factory.py | 1 - .../langchain/llms/langchain_llm_factory.py | 10 +++--- .../langchain/llms/langchain_llm_resources.py | 22 ++++++------- ..._llm_client.py => openai_client_policy.py} | 6 ++-- .../llms/standard_langchain_llm_factory.py | 32 +++++++++---------- 9 files changed, 45 insertions(+), 46 deletions(-) rename neuro_san/internals/run_context/langchain/llms/{anthropic_langchain_llm_client.py => anthropic_client_policy.py} (86%) rename neuro_san/internals/run_context/langchain/llms/{azure_langchain_llm_client.py => azure_client_policy.py} (95%) rename neuro_san/internals/run_context/langchain/llms/{bedrock_langchain_llm_client.py => bedrock_client_policy.py} (84%) rename neuro_san/internals/run_context/langchain/llms/{langchain_llm_client.py => client_policy.py} (99%) rename neuro_san/internals/run_context/langchain/llms/{openai_langchain_llm_client.py => openai_client_policy.py} (95%) diff --git a/neuro_san/internals/run_context/langchain/llms/anthropic_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/anthropic_client_policy.py similarity index 86% rename from neuro_san/internals/run_context/langchain/llms/anthropic_langchain_llm_client.py rename to neuro_san/internals/run_context/langchain/llms/anthropic_client_policy.py index d5a57d81f..570b24571 100644 --- a/neuro_san/internals/run_context/langchain/llms/anthropic_langchain_llm_client.py +++ b/neuro_san/internals/run_context/langchain/llms/anthropic_client_policy.py @@ -14,12 +14,12 @@ from contextlib import suppress -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy -class AnthropicLangChainLlmClient(LangChainLlmClient): +class AnthropicClientPolicy(ClientPolicy): """ - Implementation of the LangChainLlmClient for Anthtropic chat models. + Implementation of the ClientPolicy for Anthtropic chat models. Anthropic chat models do not allow for passing in an externally managed async web client. diff --git a/neuro_san/internals/run_context/langchain/llms/azure_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/azure_client_policy.py similarity index 95% rename from neuro_san/internals/run_context/langchain/llms/azure_langchain_llm_client.py rename to neuro_san/internals/run_context/langchain/llms/azure_client_policy.py index c8b06abec..950b04fe5 100644 --- a/neuro_san/internals/run_context/langchain/llms/azure_langchain_llm_client.py +++ b/neuro_san/internals/run_context/langchain/llms/azure_client_policy.py @@ -14,12 +14,12 @@ from leaf_common.config.resolver import Resolver -from neuro_san.internals.run_context.langchain.llms.openai_langchain_llm_client import OpenAILangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.openai_client_policy import OpenAIClientPolicy -class AzureLangChainLlmClient(OpenAILangChainLlmClient): +class AzureClientPolicy(OpenAIClientPolicy): """ - LangChainLlmClient implementation for OpenAI via Azure. + ClientPolicy implementation for OpenAI via Azure. OpenAI's BaseLanguageModel implementations do allow us to pass in a web client as an argument, so this implementation takes advantage of the create_client() diff --git a/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/bedrock_client_policy.py similarity index 84% rename from neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py rename to neuro_san/internals/run_context/langchain/llms/bedrock_client_policy.py index 805e3afbd..b6131db19 100644 --- a/neuro_san/internals/run_context/langchain/llms/bedrock_langchain_llm_client.py +++ b/neuro_san/internals/run_context/langchain/llms/bedrock_client_policy.py @@ -10,12 +10,12 @@ # # END COPYRIGHT -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy -class BedrockLangChainLlmClient(LangChainLlmClient): +class BedrockClientPolicy(ClientPolicy): """ - LangChainLlmClient implementation for Bedrock. + ClientPolicy implementation for Bedrock. Bedrock does not allow for passing in async web clients. As a matter of fact, all of its clients are synchronous, diff --git a/neuro_san/internals/run_context/langchain/llms/langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/client_policy.py similarity index 99% rename from neuro_san/internals/run_context/langchain/llms/langchain_llm_client.py rename to neuro_san/internals/run_context/langchain/llms/client_policy.py index 0fb4eced2..7d8c7da07 100644 --- a/neuro_san/internals/run_context/langchain/llms/langchain_llm_client.py +++ b/neuro_san/internals/run_context/langchain/llms/client_policy.py @@ -17,7 +17,7 @@ from langchain.llms.base import BaseLanguageModel -class LangChainLlmClient: +class ClientPolicy: """ Policy interface to manage the lifecycles of web clients that talk to LLM services. diff --git a/neuro_san/internals/run_context/langchain/llms/default_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/default_llm_factory.py index d88b5f577..45b94a850 100644 --- a/neuro_san/internals/run_context/langchain/llms/default_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/default_llm_factory.py @@ -311,7 +311,6 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: found_exception = None except NotImplementedError: - # Try ignoring the llm_client this factory # This allows for backwards compatibility with older LangChainLlmFactories llm: BaseLanguageModel = llm_factory.create_base_chat_model(config) if llm is not None: diff --git a/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py index 5f8766b00..68173d10e 100644 --- a/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py @@ -71,8 +71,8 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: :param config: The fully specified llm config which is a product of _create_full_llm_config() above. :return: A LangChainLlmResources instance containing - a BaseLanguageModel (can be Chat or LLM) and an LangChainLlmClient - policy object that contains all related resources + a BaseLanguageModel (can be Chat or LLM) and a ClientPolicy + object that contains all related resources necessary for managing the model run-time lifecycle. Can raise a ValueError if the config's class or model_name value is unknown to this method. @@ -80,13 +80,13 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: raise NotImplementedError def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str, - llm_client: Any = None) -> Any: + none_obj: Any = None) -> Any: """ :param config: The config dictionary to search :param key: The key for the config to look for :param env_key: The os.environ key whose value should be gotten if either the key does not exist or the value for the key is None - :param llm_client: An optional client instance. + :param none_obj: An optional client instance. If present this method will return None. Some BaseLanguageModels will take some kind of pre-made @@ -99,7 +99,7 @@ def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str, variables associated with creating that under-the-covers client to remain None when there is a client already made. """ - if llm_client is not None: + if none_obj is not None: return None value = None diff --git a/neuro_san/internals/run_context/langchain/llms/langchain_llm_resources.py b/neuro_san/internals/run_context/langchain/llms/langchain_llm_resources.py index 0622a2394..254a3ba82 100644 --- a/neuro_san/internals/run_context/langchain/llms/langchain_llm_resources.py +++ b/neuro_san/internals/run_context/langchain/llms/langchain_llm_resources.py @@ -12,39 +12,39 @@ from langchain_core.language_models.base import BaseLanguageModel -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy class LangChainLlmResources: """ Class for representing a LangChain model - together with run-time resources necessary for model usage by the service. + together with run-time policy necessary for model usage by the service. """ - def __init__(self, model: BaseLanguageModel, llm_client: LangChainLlmClient = None): + def __init__(self, model: BaseLanguageModel, client_policy: ClientPolicy = None): """ Constructor. :param model: Language model used. - :param http_client: optional httpx.AsyncClient used for model connections to LLM host. + :param client_policy: optional ClientPolicy object to manage connections to LLM host. """ self.model: BaseLanguageModel = model - self.llm_client: LangChainLlmClient = llm_client + self.client_policy: ClientPolicy = client_policy def get_model(self) -> BaseLanguageModel: """ - Get the language model + :return: the BaseLanguageModel """ return self.model - def get_llm_client(self) -> LangChainLlmClient: + def get_client_policy(self) -> ClientPolicy: """ - Get the client used by the model + :return: the ClientPolicy used by the model """ - return self.llm_client + return self.client_policy async def delete_resources(self): """ Release the run-time resources used by the model """ - if self.llm_client: - await self.llm_client.delete_resources() + if self.client_policy is not None: + await self.client_policy.delete_resources() diff --git a/neuro_san/internals/run_context/langchain/llms/openai_langchain_llm_client.py b/neuro_san/internals/run_context/langchain/llms/openai_client_policy.py similarity index 95% rename from neuro_san/internals/run_context/langchain/llms/openai_langchain_llm_client.py rename to neuro_san/internals/run_context/langchain/llms/openai_client_policy.py index aefc4f9c5..51fbd9cc8 100644 --- a/neuro_san/internals/run_context/langchain/llms/openai_langchain_llm_client.py +++ b/neuro_san/internals/run_context/langchain/llms/openai_client_policy.py @@ -19,12 +19,12 @@ from leaf_common.config.resolver import Resolver -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy -class OpenAILangChainLlmClient(LangChainLlmClient): +class OpenAIClientPolicy(ClientPolicy): """ - LangChainLlmClient implementation for OpenAI. + ClientPolicy implementation for OpenAI. OpenAI's BaseLanguageModel implementations do allow us to pass in a web client as an argument, so this implementation takes advantage of the create_client() diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py index 7357bb414..ca225e67f 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py @@ -16,11 +16,11 @@ from leaf_common.config.resolver import Resolver -from neuro_san.internals.run_context.langchain.llms.anthropic_langchain_llm_client import AnthropicLangChainLlmClient -from neuro_san.internals.run_context.langchain.llms.azure_langchain_llm_client import AzureLangChainLlmClient -from neuro_san.internals.run_context.langchain.llms.bedrock_langchain_llm_client import BedrockLangChainLlmClient -from neuro_san.internals.run_context.langchain.llms.openai_langchain_llm_client import OpenAILangChainLlmClient -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.anthropic_client_policy import AnthropicClientPolicy +from neuro_san.internals.run_context.langchain.llms.azure_client_policy import AzureClientPolicy +from neuro_san.internals.run_context.langchain.llms.bedrock_client_policy import BedrockClientPolicy +from neuro_san.internals.run_context.langchain.llms.openai_client_policy import OpenAIClientPolicy +from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy from neuro_san.internals.run_context.langchain.llms.langchain_llm_factory import LangChainLlmFactory from neuro_san.internals.run_context.langchain.llms.langchain_llm_resources import LangChainLlmResources @@ -84,7 +84,7 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: # pylint: disable=too-many-locals # Construct the LLM llm: BaseLanguageModel = None - llm_client: LangChainLlmClient = None + client_policy: ClientPolicy = None chat_class: str = config.get("class") if chat_class is not None: @@ -111,8 +111,8 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: install_if_missing="langchain-openai") # Create the policy object that allows us to manage the model run-time lifecycle - llm_client = OpenAILangChainLlmClient() - async_client: Any = llm_client.create_client(config) + client_policy = OpenAIClientPolicy() + async_client: Any = client_policy.create_client(config) # Now construct LLM chat model we will be using: llm = ChatOpenAI( @@ -182,8 +182,8 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: install_if_missing="langchain-openai") # Create the policy object that allows us to manage the model run-time lifecycle - llm_client = AzureLangChainLlmClient() - async_client: Any = llm_client.create_client(config) + client_policy = AzureClientPolicy() + async_client: Any = client_policy.create_client(config) # Prepare some more complex args openai_api_key: str = self.get_value_or_env(config, "openai_api_key", "AZURE_OPENAI_API_KEY", async_client) @@ -302,8 +302,8 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: verbose=False, ) - # Create the llm_client after the fact, with reach-in - llm_client = AnthropicLangChainLlmClient(llm) + # Create the client_policy after the fact, with reach-in + client_policy = AnthropicClientPolicy(llm) elif chat_class == "ollama": @@ -470,14 +470,14 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: verbose=False, ) - # Create the llm_client after the fact, with reach-in - llm_client = BedrockLangChainLlmClient(llm) + # Create the client_policy after the fact, with reach-in + client_policy = BedrockClientPolicy(llm) elif chat_class is None: raise ValueError(f"Class name {chat_class} for model_name {model_name} is unspecified.") else: raise ValueError(f"Class {chat_class} for model_name {model_name} is unrecognized.") - # Return the LlmResources with the llm_client that was passed in. + # Return the LlmResources with the client_policy that was passed in. # That might be None, and that's OK. - return LangChainLlmResources(llm, llm_client=llm_client) + return LangChainLlmResources(llm, client_policy=client_policy) From 81957f91db7b4642d0bf2e9e48fad9d89f671221 Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Fri, 3 Oct 2025 14:22:05 -0700 Subject: [PATCH 08/10] Refactor TestLlmFactory --- .../llms/standard_langchain_llm_factory.py | 4 +-- .../langchain/llms/test_llm_factory.py | 33 ++++++++----------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py index ca225e67f..8c74d082e 100644 --- a/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/standard_langchain_llm_factory.py @@ -19,10 +19,10 @@ from neuro_san.internals.run_context.langchain.llms.anthropic_client_policy import AnthropicClientPolicy from neuro_san.internals.run_context.langchain.llms.azure_client_policy import AzureClientPolicy from neuro_san.internals.run_context.langchain.llms.bedrock_client_policy import BedrockClientPolicy -from neuro_san.internals.run_context.langchain.llms.openai_client_policy import OpenAIClientPolicy from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy from neuro_san.internals.run_context.langchain.llms.langchain_llm_factory import LangChainLlmFactory from neuro_san.internals.run_context.langchain.llms.langchain_llm_resources import LangChainLlmResources +from neuro_san.internals.run_context.langchain.llms.openai_client_policy import OpenAIClientPolicy class StandardLangChainLlmFactory(LangChainLlmFactory): @@ -478,6 +478,6 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: else: raise ValueError(f"Class {chat_class} for model_name {model_name} is unrecognized.") - # Return the LlmResources with the client_policy that was passed in. + # Return the LlmResources with the client_policy that was created. # That might be None, and that's OK. return LangChainLlmResources(llm, client_policy=client_policy) diff --git a/tests/neuro_san/internals/run_context/langchain/llms/test_llm_factory.py b/tests/neuro_san/internals/run_context/langchain/llms/test_llm_factory.py index 3451e704c..ff6f87650 100644 --- a/tests/neuro_san/internals/run_context/langchain/llms/test_llm_factory.py +++ b/tests/neuro_san/internals/run_context/langchain/llms/test_llm_factory.py @@ -18,9 +18,10 @@ from langchain_core.language_models.base import BaseLanguageModel from langchain_openai.chat_models.base import ChatOpenAI -from neuro_san.internals.run_context.langchain.llms.langchain_llm_client import LangChainLlmClient +from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy from neuro_san.internals.run_context.langchain.llms.langchain_llm_factory import LangChainLlmFactory from neuro_san.internals.run_context.langchain.llms.langchain_llm_resources import LangChainLlmResources +from neuro_san.internals.run_context.langchain.llms.openai_client_policy import OpenAIClientPolicy class TestLlmFactory(LangChainLlmFactory): @@ -55,7 +56,7 @@ def create_base_chat_model(self, config: Dict[str, Any]) -> BaseLanguageModel: Create a LangChainLlmResources instance from the fully-specified llm config. This method is provided for backwards compatibility. - Prefer create_llm_resources_with_client() instead, + Prefer create_llm_resources() instead, as this allows server infrastructure to better account for outstanding connections to LLM providers when connections drop. @@ -67,26 +68,22 @@ def create_base_chat_model(self, config: Dict[str, Any]) -> BaseLanguageModel: """ raise NotImplementedError - def create_llm_resources_with_client(self, config: Dict[str, Any], - llm_client: LangChainLlmClient = None) -> LangChainLlmResources: + def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: """ Create a LangChainLlmResources instance from the fully-specified llm config. :param config: The fully specified llm config which is a product of _create_full_llm_config() above. - :param llm_client: A LangChainLlmClient instance, which by default is None, - implying that create_base_chat_model() needs to create its own client. - Note, however that a None value can lead to connection leaks and requests - that continue to run after the request connection is dropped in a server - environment. :return: A LangChainLlmResources instance containing - a BaseLanguageModel (can be Chat or LLM) and all related resources - necessary for managing the model run-time lifecycle. + a BaseLanguageModel (can be Chat or LLM) and a ClientPolicy + object necessary for managing the model run-time lifecycle. Can raise a ValueError if the config's class or model_name value is unknown to this method. """ # Construct the LLM llm: BaseLanguageModel = None + client_policy: ClientPolicy = None + chat_class: str = config.get("class") if chat_class is not None: chat_class = chat_class.lower() @@ -97,13 +94,9 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], if chat_class == "test-openai": print("Creating test-openai") - # See if there is an async_client to be had from the llm_client passed in - async_client: Any = None - if llm_client is not None: - async_openai_client = llm_client.get_client() - if async_openai_client is not None: - # Necessary reach-in - async_client = async_openai_client.chat.completions + # Create the policy object that allows us to manage the model run-time lifecycle + client_policy = OpenAIClientPolicy() + async_client: Any = client_policy.create_client(config) # Now construct LLM chat model we will be using: llm = ChatOpenAI( @@ -165,6 +158,6 @@ def create_llm_resources_with_client(self, config: Dict[str, Any], else: raise ValueError(f"Class {chat_class} for model_name {model_name} is unrecognized.") - # Return the LlmResources with the llm_client that was passed in. + # Return the LlmResources with the client_policy that was created. # That might be None, and that's OK. - return LangChainLlmResources(llm, llm_client=llm_client) + return LangChainLlmResources(llm, client_policy=client_policy) From 20146f1f0578febdbeecf2d1aa04441705ee4517 Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Mon, 6 Oct 2025 09:08:04 -0700 Subject: [PATCH 09/10] Clarify comments --- .../langchain/llms/client_policy.py | 3 ++- .../langchain/llms/langchain_llm_factory.py | 23 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/neuro_san/internals/run_context/langchain/llms/client_policy.py b/neuro_san/internals/run_context/langchain/llms/client_policy.py index 7d8c7da07..fedafd32d 100644 --- a/neuro_san/internals/run_context/langchain/llms/client_policy.py +++ b/neuro_san/internals/run_context/langchain/llms/client_policy.py @@ -78,7 +78,8 @@ def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str, :param env_key: The os.environ key whose value should be gotten if either the key does not exist or the value for the key is None :param none_obj: An optional object instance to test. - If present this method will return None. + If present this method will return None, implying + that some other external object/mechanism is supplying the values. """ if none_obj is not None: return None diff --git a/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py index 68173d10e..0c48fd06e 100644 --- a/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py @@ -87,17 +87,18 @@ def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str, :param env_key: The os.environ key whose value should be gotten if either the key does not exist or the value for the key is None :param none_obj: An optional client instance. - If present this method will return None. - - Some BaseLanguageModels will take some kind of pre-made - client as part of their constructor args, but they will - also take enough args to constructor a client for themselves - under the hood when explicitly not given that client. - - Note that this does *not* actually provide any values from - the given client, rather this arg allows those constructor - variables associated with creating that under-the-covers - client to remain None when there is a client already made. + If present this method will return None, implying + that some other external object/mechanism is supplying the values. + + Some BaseLanguageModels will take some kind of pre-made + client as part of their constructor args, but they will + also take enough args to constructor a client for themselves + under the hood when explicitly not given that client. + + Note that this does *not* actually provide any values from + the given client, rather this arg allows those constructor + variables associated with creating that under-the-covers + client to remain None when there is a client already made. """ if none_obj is not None: return None From 7560186eb09c12dea6c5821bb603e8f9a6f2f08c Mon Sep 17 00:00:00 2001 From: Dan Fink Date: Mon, 6 Oct 2025 09:18:55 -0700 Subject: [PATCH 10/10] Factor out an EnvironmentConfiguration interface for easy access to the common get_value_or_env() method --- .../interfaces/environment_configuration.py | 46 +++++++++++++++++++ .../langchain/llms/client_policy.py | 31 ++----------- .../langchain/llms/langchain_llm_factory.py | 42 ++--------------- 3 files changed, 56 insertions(+), 63 deletions(-) create mode 100644 neuro_san/internals/interfaces/environment_configuration.py diff --git a/neuro_san/internals/interfaces/environment_configuration.py b/neuro_san/internals/interfaces/environment_configuration.py new file mode 100644 index 000000000..acf53e17a --- /dev/null +++ b/neuro_san/internals/interfaces/environment_configuration.py @@ -0,0 +1,46 @@ + +# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI. +# All Rights Reserved. +# Issued under the Academic Public License. +# +# You can be released from the terms, and requirements of the Academic Public +# License by purchasing a commercial license. +# Purchase of a commercial license is mandatory for any use of the +# neuro-san SDK Software in commercial settings. +# +# END COPYRIGHT +from typing import Any +from typing import Dict + +import os + + +class EnvironmentConfiguration: + """ + Easy policy add on for the get_value_or_env() method for various classes that + are effected by configuration via dictionary/hocon and/or environment variables. + """ + + @staticmethod + def get_value_or_env(config: Dict[str, Any], key: str, env_key: str, + none_obj: Any = None) -> Any: + """ + :param config: The config dictionary to search + :param key: The key for the config to look for + :param env_key: The os.environ key whose value should be gotten if either + the key does not exist or the value for the key is None + :param none_obj: An optional object instance to test. + If present this method will return None, implying + that some other external object/mechanism is supplying the values. + """ + if none_obj is not None: + return None + + value = None + if config is not None: + value = config.get(key) + + if value is None and env_key is not None: + value = os.getenv(env_key) + + return value diff --git a/neuro_san/internals/run_context/langchain/llms/client_policy.py b/neuro_san/internals/run_context/langchain/llms/client_policy.py index fedafd32d..5f75cec61 100644 --- a/neuro_san/internals/run_context/langchain/llms/client_policy.py +++ b/neuro_san/internals/run_context/langchain/llms/client_policy.py @@ -12,14 +12,16 @@ from typing import Any from typing import Dict -import os - from langchain.llms.base import BaseLanguageModel +from neuro_san.internals.interfaces.environment_configuration import EnvironmentConfiguration + -class ClientPolicy: +class ClientPolicy(EnvironmentConfiguration): """ Policy interface to manage the lifecycles of web clients that talk to LLM services. + This inherits from EnvironmentConfiguration in order to support easy access to the + get_value_or_env() method. There are really two styles of implementation encompassed by this one interface. @@ -69,26 +71,3 @@ async def delete_resources(self): any web client references in there. """ raise NotImplementedError - - def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str, - none_obj: Any = None) -> Any: - """ - :param config: The config dictionary to search - :param key: The key for the config to look for - :param env_key: The os.environ key whose value should be gotten if either - the key does not exist or the value for the key is None - :param none_obj: An optional object instance to test. - If present this method will return None, implying - that some other external object/mechanism is supplying the values. - """ - if none_obj is not None: - return None - - value = None - if config is not None: - value = config.get(key) - - if value is None and env_key is not None: - value = os.getenv(env_key) - - return value diff --git a/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py b/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py index 0c48fd06e..4bc7b0578 100644 --- a/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py +++ b/neuro_san/internals/run_context/langchain/llms/langchain_llm_factory.py @@ -13,16 +13,17 @@ from typing import Any from typing import Dict -import os - from langchain_core.language_models.base import BaseLanguageModel +from neuro_san.internals.interfaces.environment_configuration import EnvironmentConfiguration from neuro_san.internals.run_context.langchain.llms.langchain_llm_resources import LangChainLlmResources -class LangChainLlmFactory: +class LangChainLlmFactory(EnvironmentConfiguration): """ - Interface for Factory classes creating LLM BaseLanguageModels + Interface for Factory classes creating LLM BaseLanguageModels. + This derives from EnvironmentConfiguration in order to support easy access to + the get_value_or_env() method. Most methods take a config dictionary which consists of the following keys: @@ -78,36 +79,3 @@ def create_llm_resources(self, config: Dict[str, Any]) -> LangChainLlmResources: unknown to this method. """ raise NotImplementedError - - def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str, - none_obj: Any = None) -> Any: - """ - :param config: The config dictionary to search - :param key: The key for the config to look for - :param env_key: The os.environ key whose value should be gotten if either - the key does not exist or the value for the key is None - :param none_obj: An optional client instance. - If present this method will return None, implying - that some other external object/mechanism is supplying the values. - - Some BaseLanguageModels will take some kind of pre-made - client as part of their constructor args, but they will - also take enough args to constructor a client for themselves - under the hood when explicitly not given that client. - - Note that this does *not* actually provide any values from - the given client, rather this arg allows those constructor - variables associated with creating that under-the-covers - client to remain None when there is a client already made. - """ - if none_obj is not None: - return None - - value = None - if config is not None: - value = config.get(key) - - if value is None and env_key is not None: - value = os.getenv(env_key) - - return value