Building Test Suites

1
import opik
2
3
opik_client = opik.Opik()
4
5
suite = opik_client.get_or_create_test_suite(
6
    name="customer-support-qa",
7
    project_name="test-suites-demo",
8
    global_assertions=[
9
        "The response is grounded in the provided documentation context",
10
        "The response directly addresses the user's question",
11
        "The response is concise (3 sentences or fewer)",
12
    ],
13
    global_execution_policy={"runs_per_item": 2, "pass_threshold": 2},
14
)

1
suite.insert([
2
    {
3
        "data": {
4
            "question": "How do I create a new project?",
5
            "context": "To create a new project, go to the Dashboard and click 'New Project'.",
6
        },
7
    },
8
    {
9
        "data": {
10
            "question": "Can I use this with Kubernetes?",
11
            "context": "We support Docker containers and serverless functions.",
12
        },
13
        "assertions": [
14
            "The response does NOT claim Kubernetes is supported",
15
            "The response acknowledges that the information is not available",
16
        ],
17
        "execution_policy": {"runs_per_item": 3, "pass_threshold": 2},
18
    },
19
])

1
from openai import OpenAI
2
from opik.integrations.openai import track_openai
3
4
openai_client = track_openai(OpenAI())
5
6
def make_task(system_prompt):
7
    def task(item):
8
        response = openai_client.chat.completions.create(
9
            model="gpt-4o-mini",
10
            messages=[
11
                {"role": "system", "content": system_prompt},
12
                {"role": "user", "content": f"Question: {item['question']}\n\nContext:\n{item['context']}"},
13
            ],
14
        )
15
        return {"input": item, "output": response.choices[0].message.content}
16
    return task
17
18
PROMPT_V1 = "You are a helpful assistant. Be as detailed as possible."
19
PROMPT_V2 = "You are a concise assistant. Answer based ONLY on the provided context."
20
21
result_v1 = opik.run_tests(test_suite=suite, task=make_task(PROMPT_V1))
22
result_v2 = opik.run_tests(test_suite=suite, task=make_task(PROMPT_V2))
23
24
print(f"v1 pass rate: {result_v1.pass_rate:.0%}")
25
print(f"v2 pass rate: {result_v2.pass_rate:.0%}")

1
suite.update_test_settings(
2
    global_assertions=[
3
        "The response is grounded in the provided context",
4
        "The response is concise",
5
    ],
6
    global_execution_policy={"runs_per_item": 5, "pass_threshold": 3},
7
)

1
items = suite.get_items()
2
assertions = suite.get_global_assertions()
3
policy = suite.get_global_execution_policy()
4
5
print(f"Items: {len(items)}")
6
print(f"Assertions: {assertions}")
7
print(f"Policy: {policy}")

1
items = suite.get_items()
2
suite.delete([items[0]["id"]])

1
suite = opik_client.get_or_create_test_suite(
2
    name="flaky-output-tests",
3
    global_assertions=["Response follows the expected format"],
4
    global_execution_policy={"runs_per_item": 3, "pass_threshold": 2},
5
)

1
suite.insert([{
2
    "data": {"question": "Is my account compromised?", "context": "..."},
3
    "assertions": ["Response treats the concern with urgency"],
4
    "execution_policy": {"runs_per_item": 5, "pass_threshold": 4},
5
}])