GEPA Optimizer

1
"""
2
Optimize a simple system prompt on the tiny_test dataset.
3
Requires: pip install gepa, and a valid OPENAI_API_KEY for LiteLLM-backed models.
4
"""
5
from typing import Any, Dict
6
7
from opik.evaluation.metrics import LevenshteinRatio
8
from opik.evaluation.metrics.score_result import ScoreResult
9
10
from opik_optimizer import ChatPrompt, datasets
11
from opik_optimizer.gepa_optimizer import GepaOptimizer
12
13
def levenshtein_ratio(dataset_item: Dict[str, Any], llm_output: str) -> ScoreResult:
14
    return LevenshteinRatio().score(reference=dataset_item["label"], output=llm_output)
15
16
dataset = datasets.tiny_test()
17
18
prompt = ChatPrompt(
19
    system="You are a helpful assistant. Answer concisely with the exact answer.",
20
    user="{text}",
21
)
22
23
optimizer = GepaOptimizer(
24
    model="openai/gpt-4o-mini",
25
    reflection_model="openai/gpt-4o",  # stronger reflector is often helpful
26
    project_name="GEPA_TinyTest",
27
    temperature=0.2,
28
    max_tokens=200,
29
)
30
31
result = optimizer.optimize_prompt(
32
    prompt=prompt,
33
    dataset=dataset,
34
    metric=levenshtein_ratio,
35
    max_metric_calls=12,
36
    reflection_minibatch_size=2,
37
    n_samples=5,
38
)
39
40
result.display()

1
from typing import Any, Dict
2
3
import opik
4
from opik_optimizer import ChatPrompt
5
from opik_optimizer.gepa_optimizer import GepaOptimizer
6
from opik_optimizer.datasets import hotpot_300
7
from opik_optimizer.utils import search_wikipedia
8
9
from opik.evaluation.metrics import LevenshteinRatio
10
from opik.evaluation.metrics.score_result import ScoreResult
11
12
dataset = hotpot_300()
13
14
def levenshtein_ratio(dataset_item: Dict[str, Any], llm_output: str) -> ScoreResult:
15
    metric = LevenshteinRatio()
16
    return metric.score(reference=dataset_item["answer"], output=llm_output)
17
18
prompt = ChatPrompt(
19
    system="Answer the question",
20
    user="{question}",
21
    tools=[
22
        {
23
            "type": "function",
24
            "function": {
25
                "name": "search_wikipedia",
26
                "description": "This function is used to search wikipedia abstracts.",
27
                "parameters": {
28
                    "type": "object",
29
                    "properties": {
30
                        "query": {
31
                            "type": "string",
32
                            "description": "The query parameter is the term or phrase to search for.",
33
                        },
34
                    },
35
                    "required": ["query"],
36
                },
37
            },
38
        },
39
    ],
40
    function_map={"search_wikipedia": opik.track(type="tool")(search_wikipedia)},
41
)
42
43
optimizer = GepaOptimizer(
44
    model="openai/gpt-4o-mini",   # task model
45
    reflection_model="openai/gpt-4o",  # reflection model
46
    project_name="GEPA-Hotpot",
47
    temperature=0.7,
48
    max_tokens=400,
49
)
50
51
result = optimizer.optimize_prompt(
52
    prompt=prompt,
53
    dataset=dataset,
54
    metric=levenshtein_ratio,
55
    max_metric_calls=60,
56
    reflection_minibatch_size=5,
57
    candidate_selection_strategy="best",
58
    n_samples=12,
59
)
60
61
result.display()

1
GepaOptimizer(
2
    model: str,
3
    project_name: str | None = None,
4
    reflection_model: str | None = None,  # defaults to model if None
5
    verbose: int = 1,
6
    **model_kwargs,
7
)

1
opt_result = optimizer.optimize_prompt(
2
    prompt: ChatPrompt,
3
    dataset: opik.Dataset,
4
    metric: Callable[[dict, str], ScoreResult],
5
    max_metric_calls: int,
6
    reflection_minibatch_size: int = 3,
7
    candidate_selection_strategy: str = "pareto",  # e.g., "pareto" or "best"
8
    n_samples: int | None = None,
9
)