Aligning judge metrics with human scores

import opik
PROJECT = "judge-metrics-tutorial"
PROMPT_NAME = "support-scope-judge"
JUDGE_MODEL = "openai/gpt-4o"  # the same judge model the defining guide uses
HUMAN_NAME = "support_scope_human"
QUEUE_NAME = "Support scope: review"
client = opik.Opik(project_name=PROJECT)
queue = next(q for q in client.get_traces_annotation_queues(project_name=PROJECT) if q.name == QUEUE_NAME)
rows = []
for trace in queue.get_items():
    human = next((s for s in trace.feedback_scores or [] if s.name == HUMAN_NAME), None)
    if human is None:
        continue  # not scored yet
    rows.append({
        "input": trace.input["input"],
        "output": trace.output["output"],
        "human_score": human.value,
    })
dataset = client.get_or_create_dataset(name="support-scope-alignment", project_name=PROJECT)
dataset.clear()  # rebuild from the current scores on each run
dataset.insert(rows)
print(f"{len(rows)} items in support-scope-alignment")

import json
from opik.evaluation.metrics import score_result
from opik_optimizer import ChatPrompt, MetaPromptOptimizer
def parse_score(reply: str) -> float:
    data = json.loads(reply[reply.index("{"): reply.rindex("}") + 1])
    return max(0.0, min(1.0, float(data["score"])))
def agreement(dataset_item, llm_output):
    judged = parse_score(llm_output)
    human = float(dataset_item["human_score"])
    return score_result.ScoreResult(
        name="agreement",
        value=1.0 - abs(judged - human),
        reason=f"judge {judged:.2f} vs human {human:.2f}",
    )
library = client.get_chat_prompt(name=PROMPT_NAME, project_name=PROJECT)
system_text = next(m["content"] for m in library.template if m["role"] == "system")
user_text = next(m["content"] for m in library.template if m["role"] == "user")
starting_prompt = ChatPrompt(
    system=system_text,   # the output format: not edited
    user=user_text,       # the rubric: the optimizer edits this
    model=JUDGE_MODEL,    # the judge that scores each candidate
)
optimizer = MetaPromptOptimizer(
    model="openai/gpt-5",                       # writes candidate rubrics; any strong model works
    task_context_columns=["input", "output"],   # the message and reply only: the scores would leak into the rubric
)
optimized = optimizer.optimize_prompt(
    project_name=PROJECT,
    prompt=starting_prompt,
    dataset=dataset,
    metric=agreement,
    optimize_prompts="user",
    max_trials=5,  # raise this for a longer search once the loop works
)
print(f"agreement {optimized.initial_score:.3f} -> {optimized.score:.3f}")

new_user = optimized.prompt.user
if optimized.score > optimized.initial_score and "{input}" in new_user and "{output}" in new_user:
    client.create_chat_prompt(
        name=PROMPT_NAME,
        messages=[
            {"role": "system", "content": system_text},
            {"role": "user", "content": new_user},
        ],
        change_description=f"Optimized toward human scores: {optimized.initial_score:.3f} -> {optimized.score:.3f}",
        project_name=PROJECT,
    )
    print("Saved a new version of support-scope-judge.")
else:
    print("Not saved: agreement did not improve, or the placeholders are missing.")

from opik import evaluate_experiment
latest = client.get_chat_prompt(name=PROMPT_NAME, project_name=PROJECT)  # the saved version, if one was saved
optimized_judge = RubricJudge(messages=latest.template, name="support_scope_optimized")
evaluate_experiment(
    experiment_name="support-scope-judge-v1",
    scoring_metrics=[optimized_judge],
    project_name=PROJECT,
)

gaps = {"support_scope": [], "support_scope_optimized": []}
for trace in queue.get_items():
    scores = {s.name: s.value for s in trace.feedback_scores or []}
    if HUMAN_NAME not in scores:
        continue
    for name in gaps:
        if name in scores:
            gaps[name].append(abs(scores[name] - scores[HUMAN_NAME]))
for name, values in gaps.items():
    print(f"{name}: mean |judge - human| = {sum(values) / len(values):.3f} over {len(values)} traces")

print(new_user.replace("{input}", "{{input}}").replace("{output}", "{{output}}"))