Evaluate LLM Applications with Ragas Metrics in Opik

1
import os
2
import getpass
3
4
if "OPENAI_API_KEY" not in os.environ:
5
    os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter your OpenAI API key: ")

1
# Import the required dependencies
2
from ragas.metrics import AnswerRelevancy
3
from langchain_openai.chat_models import ChatOpenAI
4
from langchain_openai.embeddings import OpenAIEmbeddings
5
from ragas.llms import LangchainLLMWrapper
6
from ragas.embeddings import LangchainEmbeddingsWrapper
7
from opik.evaluation.metrics import RagasMetricWrapper
8
9
# Initialize the Ragas metric
10
llm = LangchainLLMWrapper(ChatOpenAI())
11
emb = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
12
ragas_answer_relevancy = AnswerRelevancy(llm=llm, embeddings=emb)
13
14
# Wrap the Ragas metric with RagasMetricWrapper for Opik integration
15
answer_relevancy_metric = RagasMetricWrapper(
16
    ragas_answer_relevancy,
17
    track=True  # This enables automatic tracing in Opik
18
)

1
from opik import track
2
from opik.opik_context import update_current_trace
3
4
@track
5
def retrieve_contexts(question):
6
    # Define the retrieval function, in this case we will hard code the contexts
7
    return ["Paris is the capital of France.", "Paris is in France."]
8
9
@track
10
def answer_question(question, contexts):
11
    # Define the answer function, in this case we will hard code the answer
12
    return "Paris"
13
14
@track
15
def rag_pipeline(question):
16
    # Define the pipeline
17
    contexts = retrieve_contexts(question)
18
    answer = answer_question(question, contexts)
19
20
    # Score the pipeline using the RagasMetricWrapper
21
    score_result = answer_relevancy_metric.score(
22
        user_input=question,
23
        response=answer,
24
        retrieved_contexts=contexts
25
    )
26
27
    # Add the score to the current trace
28
    update_current_trace(
29
        feedback_scores=[{"name": score_result.name, "value": score_result.value}]
30
    )
31
32
    return answer
33
34
print(rag_pipeline("What is the capital of France?"))

1
from datasets import load_dataset
2
import opik
3
4
opik_client = opik.Opik()
5
6
# Create a small dataset
7
fiqa_eval = load_dataset("explodinggradients/fiqa", "ragas_eval")
8
9
# Reformat the dataset to match the schema expected by the Ragas evaluate function
10
hf_dataset = fiqa_eval["baseline"].select(range(3))
11
dataset_items = hf_dataset.map(
12
    lambda x: {
13
        "user_input": x["question"],
14
        "reference": x["ground_truths"][0],
15
        "retrieved_contexts": x["contexts"],
16
    }
17
)
18
dataset = opik_client.get_or_create_dataset("ragas-demo-dataset")
19
dataset.insert(dataset_items)

1
# Create an evaluation task
2
def evaluation_task(x):
3
    return {
4
        "user_input": x["question"],
5
        "response": x["answer"],
6
        "retrieved_contexts": x["contexts"],
7
    }

1
# Use the RagasMetricWrapper directly with Opik's evaluate function
2
opik.evaluation.evaluate(
3
    dataset,
4
    evaluation_task,
5
    scoring_metrics=[answer_relevancy_metric],
6
    task_threads=1,
7
)

1
from datasets import load_dataset
2
from opik.integrations.langchain import OpikTracer
3
from ragas.metrics import context_precision, answer_relevancy, faithfulness
4
from ragas import evaluate
5
6
fiqa_eval = load_dataset("explodinggradients/fiqa", "ragas_eval")
7
8
# Reformat the dataset to match the schema expected by the Ragas evaluate function
9
dataset = fiqa_eval["baseline"].select(range(3))
10
11
dataset = dataset.map(
12
    lambda x: {
13
        "user_input": x["question"],
14
        "reference": x["ground_truths"][0],
15
        "retrieved_contexts": x["contexts"],
16
    }
17
)
18
19
opik_tracer_eval = OpikTracer(tags=["ragas_eval"], metadata={"evaluation_run": True})
20
21
result = evaluate(
22
    dataset,
23
    metrics=[context_precision, faithfulness, answer_relevancy],
24
    callbacks=[opik_tracer_eval],
25
)
26
27
print(result)

1
from ragas.metrics import AnswerRelevancy
2
from langchain_openai.chat_models import ChatOpenAI
3
from langchain_openai.embeddings import OpenAIEmbeddings
4
from ragas.llms import LangchainLLMWrapper
5
from ragas.embeddings import LangchainEmbeddingsWrapper
6
from opik.evaluation.metrics import RagasMetricWrapper
7
8
# Initialize the Ragas metric
9
llm = LangchainLLMWrapper(ChatOpenAI())
10
emb = LangchainEmbeddingsWrapper(OpenAIEmbeddings())
11
12
ragas_answer_relevancy = AnswerRelevancy(llm=llm, embeddings=emb)

1
# Create the answer relevancy scoring metric
2
answer_relevancy = RagasMetricWrapper(
3
    ragas_answer_relevancy,
4
    track=True  # Enable tracing for the metric computation
5
)

1
from opik.evaluation import evaluate
2
3
evaluation_task = evaluate(
4
    dataset=dataset,
5
    task=evaluation_task,
6
    scoring_metrics=[answer_relevancy],
7
    nb_samples=10,
8
)