Making the python debugger work in VS Code with deepeval test run

[oliverbock]

The deepeval documentation recommends against running Python directly and instead suggests using deepval test run. Does anyone know how to configure launch.json to make that work?

(I'm getting the error below when running the script below that, where I'm trying to get tests to evaluate in parallel.)

        if test_cases and metrics:
>           for test_case in test_cases:
                             ^^^^^^^^^^
E           TypeError: 'EvaluationDataset' object is not iterable

.venv\Lib\site-packages\deepeval\evaluate\utils.py:361: TypeError

from deepeval.dataset import EvaluationDataset
from dotenv import load_dotenv
from deepeval import assert_test, evaluate
from deepeval.metrics import GEval
from deepeval.test_case import LLMTestCase, LLMTestCaseParams
from any_llm import completion
import sys

load_dotenv()

def test_case():
    # Check for command line argument 'bob'
    if len(sys.argv) > 1 and sys.argv[1] == 'bob':
        print("Hello Bob! Running test with special configuration.")
    else:
        print("Running standard test configuration.")
    retrieval_context1 = "All customers are eligible for a 30 day full refund at no extra costs."
    retrieval_context2 = "All customers are eligible for a 7 day full refund at no extra costs."
    input = "What if these shoes don't fit?"
    response1 = completion(
        model="gpt-5-mini",
        provider="openai",
        messages=[
            {"role": "system", "content": retrieval_context1},
            {"role": "user", "content": input},
        ]
    )
    response2 = completion(
        model="gpt-5-mini",
        provider="openai",
        messages=[
            {"role": "system", "content": retrieval_context2},
            {"role": "user", "content": input},
        ]
    )

    correctness_metric = GEval(
        name="Correctness",
        criteria="Determine if the 'actual output' is correct based on the 'expected output'.",
        evaluation_params=[LLMTestCaseParams.ACTUAL_OUTPUT, LLMTestCaseParams.EXPECTED_OUTPUT],
        threshold=0.5
    )

    test_case1 = LLMTestCase(
        input=input,
        # Replace this with the actual output from your LLM application
        actual_output=response1.choices[0].message.content,
        expected_output="We offer a 30-day full refund at no extra costs.",
        retrieval_context=[retrieval_context1]
    )
    test_case2 = LLMTestCase(
        input=input,
        # Replace this with the actual output from your LLM application
        actual_output=response2.choices[0].message.content,
        expected_output="We offer a 7-day full refund at no extra costs.",
        retrieval_context=[retrieval_context2]
    )

    # assert_test(test_case1, [correctness_metric])
    # assert_test(test_case2, [correctness_metric])
    dataset = EvaluationDataset()
    dataset.add_test_case(test_case1) # convert golden to test case
    dataset.add_test_case(test_case2) # convert golden to test case
    cases = dataset.test_cases
    evaluate(dataset,[correctness_metric])

[kritinv]

Hey @oliverbock it seems that you're passing in dataset instead of cases. Since dataset isn't a list of test cases, and is an EvaluationDataset, you're running into an object no iterable error.