Parameter Optimizer: Bayesian Parameter Tuning

1
from opik_optimizer import ParameterOptimizer
2
from opik_optimizer.parameter_optimizer import ParameterSearchSpace
3
4
optimizer = ParameterOptimizer(
5
    model="openai/gpt-4",
6
    default_n_trials=20,    # Number of optimization trials
7
    n_threads=4,            # Parallel evaluation threads
8
    seed=42
9
)

1
optimizer = ParameterOptimizer(
2
    model="openai/gpt-4",
3
    default_n_trials=50,        # More trials for thorough optimization
4
    n_threads=8,                # More parallel threads
5
    local_search_ratio=0.3,     # 30% of trials for local refinement
6
    local_search_scale=0.2,     # Scale of local search range
7
    seed=42,
8
    verbose=1                   # Verbosity level (0=off, 1=info, 2=debug)
9
)

1
from opik_optimizer import ParameterOptimizer, ChatPrompt
2
from opik_optimizer.parameter_optimizer import ParameterSearchSpace
3
from opik.evaluation.metrics import LevenshteinRatio
4
from opik_optimizer import datasets
5
6
# Initialize optimizer
7
optimizer = ParameterOptimizer(
8
    model="openai/gpt-4o-mini",
9
    default_n_trials=30,
10
    n_threads=8,
11
    seed=42
12
)
13
14
# Prepare dataset
15
dataset = datasets.hotpot_300()
16
17
# Define metric
18
def levenshtein_ratio(dataset_item, llm_output):
19
    return LevenshteinRatio().score(reference=dataset_item["answer"], output=llm_output)
20
21
# Define prompt (this stays unchanged)
22
prompt = ChatPrompt(
23
    project_name="parameter-optimization",
24
    messages=[
25
        {"role": "system", "content": "You are a helpful assistant."},
26
        {"role": "user", "content": "{question}"}
27
    ]
28
)
29
30
# Define parameter search space
31
parameter_space = ParameterSearchSpace(
32
    parameters=[
33
        {
34
            "name": "temperature",
35
            "distribution": "float",
36
            "low": 0.0,
37
            "high": 2.0
38
        },
39
        {
40
            "name": "top_p",
41
            "distribution": "float",
42
            "low": 0.1,
43
            "high": 1.0
44
        }
45
    ]
46
)
47
48
# Run optimization
49
results = optimizer.optimize_parameter(
50
    prompt=prompt,
51
    dataset=dataset,
52
    metric=levenshtein_ratio,
53
    parameter_space=parameter_space,
54
    n_samples=100
55
)
56
57
# Access results
58
results.display()
59
print(f"Best temperature: {results.details['optimized_parameters']['temperature']}")
60
print(f"Best top_p: {results.details['optimized_parameters']['top_p']}")
61
print(f"Parameter importance: {results.details['parameter_importance']}")

1
# Optimize more parameters including model selection
2
parameter_space = ParameterSearchSpace(
3
    parameters=[
4
        {
5
            "name": "temperature",
6
            "distribution": "float",
7
            "low": 0.0,
8
            "high": 1.5,
9
            "step": 0.05  # Optional: quantize values
10
        },
11
        {
12
            "name": "top_p",
13
            "distribution": "float",
14
            "low": 0.5,
15
            "high": 1.0
16
        },
17
        {
18
            "name": "frequency_penalty",
19
            "distribution": "float",
20
            "low": -1.0,
21
            "high": 1.0
22
        },
23
        {
24
            "name": "presence_penalty",
25
            "distribution": "float",
26
            "low": -1.0,
27
            "high": 1.0
28
        },
29
        {
30
            "name": "model",
31
            "distribution": "categorical",
32
            "choices": ["openai/gpt-4o-mini", "openai/gpt-4o", "openai/gpt-4-turbo"]
33
        }
34
    ]
35
)
36
37
# Run with more trials and custom Optuna sampler
38
import optuna
39
40
results = optimizer.optimize_parameter(
41
    prompt=prompt,
42
    dataset=dataset,
43
    metric=levenshtein_ratio,
44
    parameter_space=parameter_space,
45
    n_trials=100,  # Override default_n_trials
46
    n_samples=150,
47
    sampler=optuna.samplers.TPESampler(seed=42, n_startup_trials=20)
48
)

1
{
2
    "name": "temperature",
3
    "distribution": "float",
4
    "low": 0.0,
5
    "high": 2.0,
6
    "step": 0.1,      # Optional: quantize to 0.1 increments
7
    "log": False      # Optional: use log scale for sampling
8
}

1
{
2
    "name": "max_tokens",
3
    "distribution": "int",
4
    "low": 100,
5
    "high": 4000,
6
    "step": 100,      # Optional: sample in steps of 100
7
    "log": False      # Optional: use log scale
8
}

1
{
2
    "name": "model",
3
    "distribution": "categorical",
4
    "choices": ["gpt-4o-mini", "gpt-4o", "claude-3-haiku"]
5
}

1
{
2
    "name": "stream",
3
    "distribution": "bool"
4
}

1
{
2
    "name": "model_kwargs.response_format.type",
3
    "distribution": "categorical",
4
    "choices": ["text", "json_object"]
5
}

1
optimizer = ParameterOptimizer(
2
    model="openai/gpt-4o-mini",  # Using OpenAI via LiteLLM
3
    default_n_trials=30,
4
    n_threads=8
5
)