feature: disable optional imports by default

.github/workflows/lint.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install .[tests]
       - name: Pylint
         run: |
           pylint -v garak

.github/workflows/maintain_cache.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install .[all_plugins]
       - name: Build a local cache
         run: |
           export TZ=UTC

.github/workflows/remote_package_install.yml

@@ -37,7 +37,7 @@ jobs:
       - name: pip install from repo
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -U git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}
+          python -m pip install -U "git+https://github.com/${GITHUB_REPOSITORY}.git@${GITHUB_SHA}"
       - name: Sanity Test
         run: |
           python -m garak --target_type test.Blank --probes test.Test

.github/workflows/test_linux.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

.github/workflows/test_macos.yml

@@ -53,7 +53,7 @@ jobs:
           brew install libmagic
           cd garak
           python -m pip install --upgrade pip
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

.github/workflows/test_windows.yml

@@ -48,7 +48,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           cd garak
-          pip install --no-cache-dir -r requirements.txt
+          pip install --no-cache-dir .[tests]
           python -m pip cache purge
 
       - name: Restore test cache artifacts

docs/README.md

@@ -4,8 +4,10 @@
 
 1. Install dependencies:
 
+   From garak repository root: 
+
    ```console
-   python3 -m pip install -r requirements.txt
+   python3 -m pip install .
    python3 -m pip install -r docs/requirements-docs.txt
    ```
 

docs/source/configurable.rst

@@ -200,8 +200,6 @@ probes and run each prompt just once:
 
 If we save this as ``latent1.yaml`` somewhere, then we can use it with ``garak --config latent1.yaml``.
 
-
-
 Using a Custom JSON Config
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -258,6 +256,26 @@ At plugin load, the plugin instance has attributes named in ``DEFAULT_PARAMS``
 automatically created, and populated with either values given in the supplied
 config, or the default.
 
+Fixed plugin parameters
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Some plugin parameters aren't intended to be altered at instantiation via config.
+These are the fixed plugin parameters, and are generally those not given in ``DEFAULT_PARAMS``.
+Descriptions of these are as follows (for a probe - other plugins are similar):
+
+* ``description`` - A short description of what the plugin does
+* ``active`` - Whether or not the plugin is active (i.e. selected) by default
+* ``doc_uri`` - Link to more information about the plugin
+* ``extended_detectors`` - Option detectors to use on probe results
+* ``extra_dependency_names`` - Extra Python modules that garka should import when instantiatng the plugin
+* ``goal`` - Brief description in imperative form of the probe's intent
+* ``modality`` - Which modalities the probe supports (as of Nov 2024 the list is ``text``, ``image``, ``audio``, ``video``, ``3d``)
+* ``parallelisable_attempts`` - Is the probe parallelisable? Recommended false if it has to use an LLM to develop attacks, particularly a local one
+* ``primary_detector`` - What detector should be used on the probe's outputs?
+* ``tags`` - List of tags applicable to the plugin, drawn from ``garak/data/tags.misp.tsv``
+* ``mod_time`` - Modification timestamp of the plugin source file used to generate this data
+
+
 .. _config_with_yaml:
 
 Configuring Plugins with YAML

docs/source/extending.rst

@@ -29,6 +29,9 @@ The recipe for writing a new plugin or plugin class isn't outlandish:
 * Start a new module inheriting from one of the base classes, e.g. :class:`garak.probes.base.Probe`
 * Override as little as possible.
 
+If you use custom modules not included in garak's default list, include these in the plugin's top-level ``extra_dependency_names`` parameter.
+Garak's plugin loader (``garak._plugins.load_plugin()``) will manage the import and inject the requested module as ``self.<module>``.
+You should also update ``pyproject.toml`` to include a clause for the plugin, and place this plugin into the the ``all_plugins`` section.
 
 Guides to writing plugins
 -------------------------

garak/_plugins.py

@@ -404,6 +404,28 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
             ) from ve
         else:
             return False
+
+    full_plugin_name = ".".join((category, module_name, plugin_class_name))
+
+    # check cache for optional imports
+    if category in PLUGIN_TYPES:
+        extra_dependency_names = PluginCache.instance()[category][full_plugin_name][
+            "extra_dependency_names"
+        ]
+        if len(extra_dependency_names) > 0:
+            absent_modules = []
+            for dependency_module_name in extra_dependency_names:
+                for (
+                    dependency_path
+                ) in [  # support both plain names and also multi-point names e.g. langchain.llms
+                    ".".join(dependency_module_name.split(".")[: n + 1])
+                    for n in range(dependency_module_name.count(".") + 1)
+                ]:
+                    if importlib.util.find_spec(dependency_path) is None:
+                        absent_modules.append(dependency_module_name)
+            if len(absent_modules):
+                _import_failed(absent_modules, full_plugin_name)
+
     module_path = f"garak.{category}.{module_name}"
     try:
         mod = importlib.import_module(module_path)
@@ -434,6 +456,7 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
         if plugin_instance is None:
             plugin_instance = klass(config_root=config_root)
             PluginProvider.storeInstance(plugin_instance, config_root)
+
     except Exception as e:
         logging.warning(
             "Exception instantiating %s.%s: %s",
@@ -448,3 +471,44 @@ def load_plugin(path, break_on_fail=True, config_root=_config) -> object:
             return False
 
     return plugin_instance
+
+
+def load_optional_module(module_name: str):
+    try:
+        m = importlib.import_module(module_name)
+    except ModuleNotFoundError:
+        requesting_module = Path(inspect.stack()[1].filename).name.replace(".py", "")
+        _import_failed([module_name], requesting_module)
+    return m
+
+
+def _import_failed(absent_modules: [str], calling_module: str):
+    quoted_module_list = "'" + "', '".join(absent_modules) + "'"
+    module_list = " ".join(absent_modules)
+    msg = f"⛔ Plugin '{calling_module}' requires Python modules which aren't installed/available: {quoted_module_list}"
+    hint = f"💡 Try 'pip install {module_list}' to get missing module."
+    logging.critical(msg)
+    print(msg + "\n" + hint)
+    raise ModuleNotFoundError(msg)
+
+
+def _load_deps(self, deps_override=list()):
+    # load external dependencies. should be invoked at construction and
+    # in _client_load (if used)
+    dep_names = deps_override if deps_override else self.extra_dependency_names
+    for extra_dependency in dep_names:
+        extra_dep_name = extra_dependency.replace(".", "_").replace("-", "_")
+        if not hasattr(self, extra_dep_name) or getattr(self, extra_dep_name) is None:
+            setattr(
+                self,
+                extra_dep_name,
+                load_optional_module(extra_dependency),
+            )
+
+
+def _clear_deps(self):
+    # unload external dependencies from class. should be invoked before
+    # serialisation, esp. in _clear_client (if used)
+    for extra_dependency in self.extra_dependency_names:
+        extra_dep_name = extra_dependency.replace(".", "_")
+        setattr(self, extra_dep_name, None)

garak/buffs/base.py

@@ -27,6 +27,8 @@ class Buff(Configurable):
     doc_uri = ""
     lang = None  # set of languages this buff should be constrained to
     active = True
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     DEFAULT_PARAMS = {}
 

garak/detectors/base.py

@@ -27,6 +27,8 @@ class Detector(Configurable):
     accuracy = None
     active = True
     tags = []  # list of taxonomy categories per the MISP format
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     # support mainstream any-to-any large models
     # legal element for str list `modality['in']`: 'text', 'image', 'audio', 'video', '3d'

garak/exception.py

@@ -14,7 +14,7 @@ class TargetNameMissingError(GarakException):
     """A generator requires target_name to be set, but it wasn't"""
 
 
-class GarakBackoffTrigger(GarakException):
+class GeneratorBackoffTrigger(GarakException):
     """Thrown when backoff should be triggered"""
 
 

garak/generators/azure.py

@@ -83,8 +83,9 @@ def _validate_env_var(self):
         return super()._validate_env_var()
 
     def _load_client(self):
-        if self.target_name in openai_model_mapping:
-            self.target_name = openai_model_mapping[self.target_name]
+        self._load_deps()
+        if self.model_name in openai_model_mapping:
+            self.model_name = openai_model_mapping[self.model_name]
 
         self.client = openai.AzureOpenAI(
             azure_endpoint=self.uri, api_key=self.api_key, api_version=self.api_version

garak/generators/base.py

@@ -10,7 +10,7 @@
 from colorama import Fore, Style
 import tqdm
 
-from garak import _config
+from garak import _config, _plugins
 from garak.attempt import Message, Conversation
 from garak.configurable import Configurable
 from garak.exception import GarakException
@@ -45,6 +45,8 @@ class Generator(Configurable):
     supports_multiple_generations = (
         False  # can more than one generation be extracted per request?
     )
+    # list of strings naming modules required but not explicitly in garak by default
+    extra_dependency_names = []
 
     def __init__(self, name="", config_root=_config):
         self._load_config(config_root)
@@ -64,6 +66,10 @@ def __init__(self, name="", config_root=_config):
             f"🦜 loading {Style.BRIGHT}{Fore.LIGHTMAGENTA_EX}generator{Style.RESET_ALL}: {self.generator_family_name}: {self.name}"
         )
         logging.info("generator init: %s", self)
+        self._load_deps()
+
+    _load_deps = _plugins._load_deps
+    _clear_deps = _plugins._clear_deps
 
     def _call_model(
         self, prompt: Conversation, generations_this_call: int = 1

garak/generators/cohere.py

@@ -15,12 +15,11 @@
 from typing import List, Union
 
 import backoff
-from cohere.core.api_error import ApiError
-import cohere
 import tqdm
 
 from garak import _config
 from garak.attempt import Message, Conversation
+from garak.exception import GeneratorBackoffTrigger
 from garak.generators.base import Generator
 
 
@@ -51,6 +50,8 @@ class CohereGenerator(Generator):
         "api_version": "v2",  # "v1" for legacy generate API, "v2" for chat API (recommended)
     }
 
+    extra_dependency_names = ["cohere"]
+
     generator_family_name = "Cohere"
 
     def __init__(self, name="command", config_root=_config):
@@ -73,11 +74,11 @@ def __init__(self, name="command", config_root=_config):
         # Initialize appropriate client based on API version
         # Following Cohere's guidance to use Client() for v1 and ClientV2() for v2
         if self.api_version == "v1":
-            self.generator = cohere.Client(api_key=self.api_key)
+            self.generator = self.cohere.Client(api_key=self.api_key)
         else:  # api_version == "v2"
-            self.generator = cohere.ClientV2(api_key=self.api_key)
+            self.generator = self.cohere.ClientV2(api_key=self.api_key)
 
-    @backoff.on_exception(backoff.fibo, ApiError, max_value=70)
+    @backoff.on_exception(backoff.fibo, GeneratorBackoffTrigger, max_value=70)
     def _call_cohere_api(self, prompt_text, request_size=COHERE_GENERATION_LIMIT):
         """Empty prompts raise API errors (e.g. invalid request: prompt must be at least 1 token long).
         We catch these using the ApiError base class in Cohere v5+.
@@ -126,9 +127,17 @@ def _call_cohere_api(self, prompt_text, request_size=COHERE_GENERATION_LIMIT):
                                 "Chat response structure doesn't match expected format"
                             )
                             responses.append(str(response))
-                    except ApiError as e:
-                        raise e  # bubble up ApiError for backoff handling
                     except Exception as e:
+
+                        backoff_exception_types = (
+                            self.cohere.errors.GatewayTimeoutError,
+                            self.cohere.errors.TooManyRequestsError,
+                            self.cohere.errors.ServiceUnavailableError,
+                            self.cohere.errors.InternalServerError,
+                        )
+                        for backoff_exception in backoff_exception_types:
+                            if isinstance(e, backoff_exception):
+                                raise GeneratorBackoffTrigger from e  # bubble up ApiError for backoff handling
                         logging.error(f"Chat API error: {e}")
                         responses.append(None)
 

garak/generators/groq.py

@@ -40,6 +40,7 @@ class GroqChat(OpenAICompatible):
     generator_family_name = "Groq"
 
     def _load_client(self):
+        self._load_deps()
         self.client = openai.OpenAI(base_url=self.uri, api_key=self.api_key)
         if self.name in ("", None):
             raise ValueError(

garak/generators/guardrails.py

@@ -17,27 +17,22 @@ class NeMoGuardrails(Generator):
 
     supports_multiple_generations = False
     generator_family_name = "Guardrails"
+    extra_dependency_names = ["nemoguardrails"]
 
     def __init__(self, name="", config_root=_config):
-        # another class that may need to skip testing due to non required dependency
-        try:
-            from nemoguardrails import RailsConfig, LLMRails
-        except ImportError as e:
-            raise NameError(
-                "You must first install NeMo Guardrails using `pip install nemoguardrails`."
-            ) from e
 
         self.name = name
         self._load_config(config_root)
         self.fullname = f"Guardrails {self.name}"
 
-        # Currently, we use the target_name as the path to the config
-        with redirect_stderr(io.StringIO()) as f:  # quieten the tqdm
-            config = RailsConfig.from_path(self.name)
-            self.rails = LLMRails(config=config)
-
         super().__init__(self.name, config_root=config_root)
 
+        set_verbose = self.nemoguardrails.logging.verbose.set_verbose
+        # Currently, we use the model_name as the path to the config
+        with redirect_stderr(io.StringIO()) as f:  # quieten the tqdm
+            config = self.nemoguardrails.RailsConfig.from_path(self.name)
+            self.rails = self.nemoguardrails.LLMRails(config=config)
+
     def _call_model(
         self, prompt: Conversation, generations_this_call: int = 1
     ) -> List[Union[Message, None]]: