UN-3477 Client Policy for LLMs

neuro_san/internals/run_context/langchain/llms/anthropic_client_policy.py

@@ -0,0 +1,48 @@
+
+# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI.
+# All Rights Reserved.
+# Issued under the Academic Public License.
+#
+# You can be released from the terms, and requirements of the Academic Public
+# License by purchasing a commercial license.
+# Purchase of a commercial license is mandatory for any use of the
+# neuro-san SDK Software in commercial settings.
+#
+# END COPYRIGHT
+
+from typing import Any
+
+from contextlib import suppress
+
+from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy
+
+
+class AnthropicClientPolicy(ClientPolicy):
+    """
+    Implementation of the ClientPolicy for Anthtropic chat models.
+
+    Anthropic chat models do not allow for passing in an externally managed
+    async web client.
+    """
+
+    async def delete_resources(self):
+        """
+        Release the run-time resources used by the model
+        """
+        if self.llm is None:
+            return
+
+        # Do the necessary reach-ins to successfully shut down the web client
+
+        # This is really an anthropic.AsyncClient, but we don't really want to do the Resolver here.
+        # Note we don't want to do this in the constructor, as AnthropicChat lazily
+        # creates these as needed via a cached_property that needs to be done in its own time
+        # via Anthropic infrastructure.  By the time we get here, it's already been created.
+        anthropic_async_client: Any = self.llm._async_client     # pylint:disable=protected-access
+
+        if anthropic_async_client is not None:
+            with suppress(Exception):
+                await anthropic_async_client.aclose()
+
+        # Let's not do this again, shall we?
+        self.llm = None

neuro_san/internals/run_context/langchain/llms/azure_client_policy.py

@@ -0,0 +1,98 @@
+
+# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI.
+# All Rights Reserved.
+# Issued under the Academic Public License.
+#
+# You can be released from the terms, and requirements of the Academic Public
+# License by purchasing a commercial license.
+# Purchase of a commercial license is mandatory for any use of the
+# neuro-san SDK Software in commercial settings.
+#
+# END COPYRIGHT
+from typing import Any
+from typing import Dict
+
+from leaf_common.config.resolver import Resolver
+
+from neuro_san.internals.run_context.langchain.llms.openai_client_policy import OpenAIClientPolicy
+
+
+class AzureClientPolicy(OpenAIClientPolicy):
+    """
+    ClientPolicy implementation for OpenAI via Azure.
+
+    OpenAI's BaseLanguageModel implementations do allow us to pass in a web client
+    as an argument, so this implementation takes advantage of the create_client()
+    method to do that. Worth noting that where many other implementations might care about
+    the llm reference, because of our create_client() implementation, we do not.
+    """
+
+    def create_client(self, config: Dict[str, Any]) -> Any:
+        """
+        Creates the web client to used by a BaseLanguageModel to be
+        constructed in the future.  Neuro SAN infrastructures prefers that this
+        be an asynchronous client, however we realize some BaseLanguageModels
+        do not support that (even though they should!).
+
+        Implementations should retain any references to state that needs to be cleaned up
+        in the delete_resources() method.
+
+        :param config: The fully specified llm config
+        :return: The web client that accesses the LLM.
+                By default this is None, as many BaseLanguageModels
+                do not allow a web client to be passed in as an arg.
+        """
+        # OpenAI is the one chat class that we do not require any extra installs.
+        # This is what we want to work out of the box.
+        # Nevertheless, have it go through the same lazy-loading resolver rigamarole as the others.
+
+        # Set up a resolver to use to resolve lazy imports of classes from
+        # langchain_* packages to prevent installing the world.
+        resolver = Resolver()
+
+        # pylint: disable=invalid-name
+        AsyncAzureOpenAI = resolver.resolve_class_in_module("AsyncAzureOpenAI",
+                                                            module_name="openai",
+                                                            install_if_missing="langchain-openai")
+
+        self.create_http_client(config)
+
+        # Prepare some more complex args
+        openai_api_key: str = self.get_value_or_env(config, "openai_api_key", "AZURE_OPENAI_API_KEY")
+        if openai_api_key is None:
+            openai_api_key = self.get_value_or_env(config, "openai_api_key", "OPENAI_API_KEY")
+
+        # From lanchain_openai.chat_models.azure.py
+        default_headers: Dict[str, str] = {}
+        default_headers = config.get("default_headers", default_headers)
+        default_headers.update({
+            "User-Agent": "langchain-partner-python-azure-openai",
+        })
+
+        self.async_openai_client = AsyncAzureOpenAI(
+            azure_endpoint=self.get_value_or_env(config, "azure_endpoint",
+                                                 "AZURE_OPENAI_ENDPOINT"),
+            deployment_name=self.get_value_or_env(config, "deployment_name",
+                                                  "AZURE_OPENAI_DEPLOYMENT_NAME"),
+            api_version=self.get_value_or_env(config, "openai_api_version",
+                                              "OPENAI_API_VERSION"),
+            api_key=openai_api_key,
+            # AD here means "ActiveDirectory"
+            azure_ad_token=self.get_value_or_env(config, "azure_ad_token",
+                                                 "AZURE_OPENAI_AD_TOKEN"),
+            # azure_ad_token_provider is a complex object, and we can't set that through config
+
+            organization=self.get_value_or_env(config, "openai_organization", "OPENAI_ORG_ID"),
+            # project           - not set in langchain_openai
+            # webhook_secret    - not set in langchain_openai
+            base_url=self.get_value_or_env(config, "openai_api_base", "OPENAI_API_BASE"),
+            timeout=config.get("request_timeout"),
+            max_retries=config.get("max_retries"),
+            default_headers=default_headers,
+            # default_query     - don't understand enough to set, but set in langchain_openai
+            http_client=self.http_client
+        )
+
+        # We retain the async_openai_client reference, but we hand back this reach-in
+        # to pass to the BaseLanguageModel constructor.
+        return self.async_openai_client.chat.completions

neuro_san/internals/run_context/langchain/llms/bedrock_client_policy.py

@@ -0,0 +1,42 @@
+
+# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI.
+# All Rights Reserved.
+# Issued under the Academic Public License.
+#
+# You can be released from the terms, and requirements of the Academic Public
+# License by purchasing a commercial license.
+# Purchase of a commercial license is mandatory for any use of the
+# neuro-san SDK Software in commercial settings.
+#
+# END COPYRIGHT
+
+from neuro_san.internals.run_context.langchain.llms.client_policy import ClientPolicy
+
+
+class BedrockClientPolicy(ClientPolicy):
+    """
+    ClientPolicy implementation for Bedrock.
+
+    Bedrock does not allow for passing in async web clients.
+    As a matter of fact, all of its clients are synchronous,
+    which is not the best for an async service.
+    """
+
+    async def delete_resources(self):
+        """
+        Release the run-time resources used by the model
+        """
+        if self.llm is None:
+            return
+
+        # Do the necessary reach-ins to successfully shut down the web client
+        if self.llm.client is not None:
+            # This is a boto3 client
+            self.llm.client.close()
+
+        if self.llm.bedrock_client is not None:
+            # This is a boto3 client
+            self.llm.bedrock_client.close()
+
+        # Let's not do this again, shall we?
+        self.llm = None

neuro_san/internals/run_context/langchain/llms/client_policy.py

@@ -0,0 +1,93 @@
+
+# Copyright (C) 2023-2025 Cognizant Digital Business, Evolutionary AI.
+# All Rights Reserved.
+# Issued under the Academic Public License.
+#
+# You can be released from the terms, and requirements of the Academic Public
+# License by purchasing a commercial license.
+# Purchase of a commercial license is mandatory for any use of the
+# neuro-san SDK Software in commercial settings.
+#
+# END COPYRIGHT
+from typing import Any
+from typing import Dict
+
+import os
+
+from langchain.llms.base import BaseLanguageModel
+
+
+class ClientPolicy:
+    """
+    Policy interface to manage the lifecycles of web clients that talk to LLM services.
+
+    There are really two styles of implementation encompassed by this one interface.
+
+    1) When BaseLanguageModels can have web clients passed into their constructor,
+       implementations should use the create_client() method to retain any references necessary
+       to help them clean up nicely in the delete_resources() method.
+
+    2) When BaseLanguageModels cannot have web clients passed into their constructor,
+       implementations should pass the already created llm into their implementation's
+       constructor. Later delete_resources() implementations will need to do a reach-in
+       to the llm instance to clean up any references related to the web client.
+    """
+
+    def __init__(self, llm: BaseLanguageModel = None):
+        """
+        Constructor.
+
+        :param llm: BaseLanguageModel
+        """
+        self.llm: BaseLanguageModel = llm
+
+    # pylint: disable=useless-return
+    def create_client(self, config: Dict[str, Any]) -> Any:
+        """
+        Creates the web client to used by a BaseLanguageModel to be
+        constructed in the future.  Neuro SAN infrastructures prefers that this
+        be an asynchronous client, however we realize some BaseLanguageModels
+        do not support that (even though they should!).
+
+        Implementations should retain any references to state that needs to be cleaned up
+        in the delete_resources() method.
+
+        :param config: The fully specified llm config
+        :return: The web client that accesses the LLM.
+                By default this is None, as many BaseLanguageModels
+                do not allow a web client to be passed in as an arg.
+        """
+        _ = config
+        return None
+
+    async def delete_resources(self):
+        """
+        Release the run-time resources used by the instance.
+
+        Unfortunately for many BaseLanguageModels, this tends to involve
+        a reach-in to its private internals in order to shutting down
+        any web client references in there.
+        """
+        raise NotImplementedError
+
+    def get_value_or_env(self, config: Dict[str, Any], key: str, env_key: str,
+                         none_obj: Any = None) -> Any:
+        """
+        :param config: The config dictionary to search
+        :param key: The key for the config to look for
+        :param env_key: The os.environ key whose value should be gotten if either
+                        the key does not exist or the value for the key is None
+        :param none_obj:  An optional object instance to test.
+                          If present this method will return None.
+        """
+        if none_obj is not None:
+            return None
+
+        value = None
+        if config is not None:
+            value = config.get(key)
+
+        if value is None and env_key is not None:
+            value = os.getenv(env_key)
+
+        return value