Define metrics

1
from opik.evaluation.metrics.score_result import ScoreResult
2
3
def short_answer(item, output):
4
    is_short = len(output) <= 200
5
    return ScoreResult(
6
        name="short_answer",
7
        value=1.0 if is_short else 0.0,
8
        reason="Answer under 200 chars" if is_short else "Answer too long"
9
    )

1
from opik_optimizer import MultiMetricObjective
2
from opik.evaluation.metrics import LevenshteinRatio, AnswerRelevance
3
4
objective = MultiMetricObjective(
5
    weights=[0.6, 0.4],
6
    metrics=[
7
        lambda item, output: LevenshteinRatio().score(reference=item["answer"], output=output),
8
        lambda item, output: AnswerRelevance().score(
9
            context=[item["answer"]], output=output, input=item["question"]
10
        ),
11
    ],
12
    name="accuracy_and_relevance",
13
)

1
from opik_optimizer import MultiMetricObjective
2
from opik.evaluation.metrics import AnswerRelevance
3
from opik_optimizer.metrics import SpanCost, SpanDuration
4
5
# Regular metric without task_span
6
def answer_relevance(dataset_item, llm_output):
7
    metric = AnswerRelevance()
8
    return metric.score(
9
        context=[dataset_item["answer"]], 
10
        output=llm_output, 
11
        input=dataset_item["question"]
12
    )
13
14
# Built-in span metrics can be normalized with target= for clean multi-metric weighting.
15
# invert=True (default) means lower raw value -> higher score.
16
cost = SpanCost(target=0.01, invert=True, name="cost_score")
17
duration = SpanDuration(target=6.0, invert=True, name="duration_score")
18
19
# Combine quality, cost, and speed metrics on a common [0, 1] scale
20
objective = MultiMetricObjective(
21
    metrics=[answer_relevance, cost, duration],
22
    weights=[0.33, 0.33, 0.33],  # equally optimize for accuracy, cost and duration/latency
23
    name="quality_cost_speed",
24
)

1
from opik_optimizer import MultiMetricObjective
2
from opik_optimizer.metrics import LevenshteinAccuracyMetric, SpanCost, SpanDuration
3
4
accuracy = LevenshteinAccuracyMetric(reference_key="answer")
5
cost = SpanCost(target=0.01, invert=True, name="cost_score")
6
duration = SpanDuration(target=6.0, invert=True, name="duration_score")
7
8
objective = MultiMetricObjective(
9
    metrics=[accuracy, cost, duration],
10
    weights=[0.5, 0.25, 0.25],  # all metrics already normalized to [0, 1]
11
    name="accuracy_cost_duration",
12
)