test(models/openrouter): add e2e tests (#2554)

* test(models/openrouter): add e2e tests

* test(models/openrouter): remove invalid models

* test(models/openrouter): try to fix test

* test(models/openrouter): idk why, but test fail on that check

* test(models/openrouter): idk why, but test fail on that

Author: Stream29

models/openrouter/manifest.yaml

@@ -26,5 +26,5 @@ type: plugin
 plugins:
   models:
     - provider/openrouter.yaml
-version: 0.0.31
+version: 0.0.32
 created_at: 2024-09-20T00:13:50.29298939-04:00

models/openrouter/models/llm/_position.yaml

@@ -8,17 +8,11 @@
 - openai/o3-mini-high
 - openai/o3-mini
 - openai/o3-mini-2025-01-31
-- openai/o1-preview
-- openai/o1-mini
-- openai/gpt-4.1
-- openai/gpt-4.1-mini
-- openai/gpt-4.1-nano
 - openai/chatgpt-4o-latest
 - openai/gpt-4o-2024-11-20
 - openai/gpt-4o
 - openai/gpt-4o-mini
 - openai/gpt-4
-- openai/gpt-4-32k
 - openai/gpt-3.5-turbo
 - anthropic/claude-opus-4.5
 - anthropic/claude-opus-4
@@ -30,30 +24,18 @@
 - anthropic/claude-3.5-sonnet
 - anthropic/claude-haiku-4.5
 - anthropic/claude-3-haiku
-- anthropic/claude-3-opus
-- anthropic/claude-3-sonnet
 - google/gemini-3-pro-preview
 - google/gemini-3-flash-preview
 - google/gemini-2.5-pro
 - google/gemini-2.5-flash-preview-09-2025
 - google/gemini-2.5-flash
-- google/gemini-2.5-flash-image-preview
-- google/gemini-2.5-flash-image-preview:free
 - google/gemini-2.5-flash-lite-preview-09-2025
 - google/gemini-2.5-flash-lite
-- google/gemini-2.5-flash-lite-preview-06-17
 - google/gemini-2.0-flash-lite-001
-- google/gemini-pro-1.5
-- google/gemini-flash-1.5
-- google/gemini-pro
 - google/gemma-3n-e2b-it:free
-- cohere/command-r-plus
-- cohere/command-r
 - meta-llama/llama-3.2-1b-instruct
 - meta-llama/llama-3.2-3b-instruct
 - meta-llama/llama-3.2-11b-vision-instruct
-- meta-llama/llama-3.2-90b-vision-instruct
-- meta-llama/llama-3.1-405b-instruct
 - meta-llama/llama-3.1-70b-instruct
 - meta-llama/llama-3.1-8b-instruct
 - meta-llama/llama-3-70b-instruct
@@ -62,27 +44,16 @@
 - mistralai/mixtral-8x7b-instruct
 - mistralai/mistral-7b-instruct
 - qwen/qwen-2.5-72b-instruct
-- qwen/qwen-2-72b-instruct
 - qwen/qwen3-max
 - deepseek/deepseek-chat
 - deepseek/deepseek-chat-v3.1
 - deepseek/deepseek-chat-v3-0324
-- deepseek/deepseek-coder
-- deepseek/deepseek-prover-v2
 - deepseek/deepseek-r1
 - deepseek/deepseek-r1-0528
-- deepseek/deepseek-r1-0528-qwen3-8b
 - deepseek/deepseek-r1-distill-llama-70b
 - deepseek/deepseek-r1-distill-qwen-32b
-- deepseek/deepseek-r1-distill-qwen-14b
-- deepseek/deepseek-r1-distill-llama-8b
-- deepseek/deepseek-r1-distill-qwen-7b
-- deepseek/deepseek-r1-distill-qwen-1.5b
-- deepseek/deepseek-v3-base
 - moonshotai/kimi-k2-0905
-- moonshotai/kimi-k2-0711
 - x-ai/grok-4.1-fast
 - x-ai/grok-4-fast
-- x-ai/grok-4-fast:free
 - x-ai/grok-3-beta
-- x-ai/grok-3-mini-beta
+- x-ai/grok-3-mini-beta

models/openrouter/pyproject.toml

@@ -14,6 +14,8 @@ dependencies = [
 # uv run black . -C -l 100 && uv run ruff check --fix
 [dependency-groups]
 dev = [
+    "pyrefly>=0.50.0",
+    "pytest>=9.0.2",
     "black>=25.11.0",
     "ruff>=0.14.6",
 ]

models/openrouter/tests/test_llm_call.py

@@ -0,0 +1,76 @@
+import os
+from pathlib import Path
+
+import pytest
+import yaml
+
+from dify_plugin.config.integration_config import IntegrationConfig
+from dify_plugin.core.entities.plugin.request import (
+    ModelActions,
+    ModelInvokeLLMRequest,
+    PluginInvokeType,
+)
+from dify_plugin.entities.model import ModelType
+from dify_plugin.entities.model.llm import LLMResultChunk
+from dify_plugin.integration.run import PluginRunner
+
+
+def get_all_models() -> list[str]:
+    models_dir = Path(__file__).parent.parent / "models" / "llm"
+    position_file = models_dir / "_position.yaml"
+    if not position_file.exists():
+        raise FileNotFoundError(f"Missing model position file: {position_file}")
+
+    try:
+        data = yaml.safe_load(position_file.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise ValueError(f"Invalid YAML in {position_file}") from exc
+
+    if data is None:
+        return []
+    if not isinstance(data, list):
+        raise ValueError(f"Expected a YAML list in {position_file}")
+
+    models: list[str] = []
+    for item in data:
+        if isinstance(item, str) and item.strip():
+            models.append(item.strip())
+    return models
+
+
+@pytest.mark.parametrize("model_name", get_all_models())
+def test_llm_invoke(model_name: str) -> None:
+    api_key = os.getenv("OPENROUTER_API_KEY")
+    if not api_key:
+        raise ValueError("OPENROUTER_API_KEY environment variable is required")
+
+    plugin_path = os.getenv("PLUGIN_FILE_PATH")
+    if not plugin_path:
+        plugin_path = str(Path(__file__).parent.parent)
+
+    payload = ModelInvokeLLMRequest(
+        user_id="test_user",
+        provider="openrouter",
+        model_type=ModelType.LLM,
+        model=model_name,
+        credentials={"api_key": api_key},
+        prompt_messages=[{"role": "user", "content": "Say hello in one word."}],
+        stop=None,
+        tools=None,
+        stream=True,
+        model_parameters={},
+    )
+
+    with PluginRunner(
+        config=IntegrationConfig(), plugin_package_path=plugin_path
+    ) as runner:
+        results: list[LLMResultChunk] = []
+        for result in runner.invoke(
+            access_type=PluginInvokeType.Model,
+            access_action=ModelActions.InvokeLLM,
+            payload=payload,
+            response_type=LLMResultChunk,
+        ):
+            results.append(result)
+
+        assert len(results) > 0, f"No results received for model {model_name}"