Quick Start

1
import { config } from "dotenv";
2
import { evaluate, EvaluationTask, Opik, ExactMatch } from "opik";
3
import OpenAI from "openai";
4
5
// Load environment variables from .env file
6
config();
7
8
// Initialize the OpenAI client
9
const client = new OpenAI();
10
11
// Create an Opik client
12
const opik = new Opik();
13
14
// define the type for DatasetItem
15
type DatasetItem = {
16
  input: string;
17
  expected_output: string;
18
  metadata: {
19
    category: string;
20
    difficulty: string;
21
    version: number;
22
  };
23
};
24
25
// Retrieve the existing dataset by name
26
const retrievedDataset = await opik.getOrCreateDataset<DatasetItem>(
27
  "testDataset"
28
);
29
30
// Add items to a dataset
31
const itemsToAdd = [
32
  {
33
    input: "What is machine learning?",
34
    expected_output:
35
      "Machine learning is a type of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed.",
36
    metadata: { category: "AI basics", difficulty: "beginner", version: 1 },
37
  },
38
];
39
await retrievedDataset.insert(itemsToAdd);
40
41
// Define a task that takes a dataset item and returns a result
42
const llmTask: EvaluationTask<DatasetItem> = async (datasetItem) => {
43
  const { input } = datasetItem;
44
45
  const response = await client.responses.create({
46
    model: "gpt-4o",
47
    instructions: "You are a coding assistant that talks like a pirate",
48
    input,
49
  });
50
51
  return { output: response.output_text };
52
};
53
54
// Run evaluation
55
const result = await evaluate({
56
  dataset: retrievedDataset,
57
  task: llmTask,
58
  scoringMetrics: [new ExactMatch()],
59
  experimentName: "My First Evaluation",
60
61
  // map the output of the task and dataset item data to the expected metric inputs
62
  scoringKeyMapping: {
63
    expected: "expected_output",
64
  },
65
});
66
67
console.log(`Experiment ID: ${result.experimentId}`);
68
console.log(`Experiment Name: ${result.experimentName}`);
69
console.log(`Total test cases: ${result.testResults.length}`);

1
import { config } from "dotenv";
2
import { evaluate, EvaluationTask, Opik, ExactMatch } from "opik";
3
import OpenAI from "openai";
4
5
// Load environment variables from .env file
6
config();
7
8
// Initialize the OpenAI client
9
const client = new OpenAI();
10
11
// Create an Opik client
12
const opik = new Opik();

1
// Create an Opik client
2
const opik = new Opik();
3
4
// define the type for DatasetItem
5
type DatasetItem = {
6
  input: string;
7
  expected_output: string;
8
  metadata: {
9
    category: string;
10
    difficulty: string;
11
    version: number;
12
  };
13
};
14
15
// Retrieve the existing dataset by name
16
const retrievedDataset = await opik.getOrCreateDataset<DatasetItem>(
17
  "testDataset"
18
);
19
20
// Add items to a dataset
21
const itemsToAdd = [
22
  {
23
    input: "What is machine learning?",
24
    expected_output:
25
      "Machine learning is a type of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed.",
26
    metadata: { category: "AI basics", difficulty: "beginner", version: 1 },
27
  },
28
];
29
await retrievedDataset.insert(itemsToAdd);

1
// Define a task that takes a dataset item and returns a result
2
const llmTask: EvaluationTask<DatasetItem> = async (datasetItem) => {
3
  const { input } = datasetItem;
4
5
  const response = await client.responses.create({
6
    model: "gpt-4o", // Use any model you need to evaluate
7
    instructions: "You are a coding assistant that talks like a pirate",
8
    input,
9
  });
10
11
  return { output: response.output_text };
12
};

1
// Run evaluation with a built-in ExactMatch metric
2
const result = await evaluate({
3
  dataset: retrievedDataset,
4
  task: llmTask,
5
  scoringMetrics: [new ExactMatch()], // Use multiple metrics for comprehensive evaluation
6
  experimentName: "My First Evaluation",
7
8
  // map the output of the task and dataset item data to the expected metric inputs
9
  scoringKeyMapping: {
10
    expected: "expected_output",
11
  },
12
});
13
14
console.log(`Experiment URL: ${result.resultUrl}`); // Direct link to view results

1
{
2
  "experimentId": "01975908-818f-765a-abv6-08d179c15610",
3
  "experimentName": "My First Evaluation",
4
  "testResults": [
5
    {
6
      "testCase": {
7
        "traceId": "01975908-82dc-73fd-862d-dd51152ddse1",
8
        "datasetItemId": "01975908-810c-7663-b7a3-e3ae94484ca9",
9
        "scoringInputs": {
10
          "input": "What is machine learning?",
11
          "metadata": {
12
            "category": "AI basics",
13
            "difficulty": "beginner",
14
            "version": 1
15
          },
16
          "expected_output": "Machine learning is a type of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed.",
17
          "id": "01975908-810c-7663-b7a3-e3ae43884ca9",
18
          "output": "Arrr, machine learnin' be a way for computers to learn from data, akin to teachin' a parrot new tricks! Instead of givin' exact instructions, ye feed the machine lots o' examples, and it figures out how to make decisions on its own. It be useful for predictin' things, findin' patterns, and even speakin' like a fine pirate! 🏴‍☠️",
19
          "expected": "Machine learning is a type of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed."
20
        },
21
        "taskOutput": {
22
          "output": "Arrr, machine learnin' be a way for computers to learn from data, akin to teachin' a parrot new tricks! Instead of givin' exact instructions, ye feed the machine lots o' examples, and it figures out how to make decisions on its own. It be useful for predictin' things, findin' patterns, and even speakin' like a fine pirate! 🏴‍☠️"
23
        }
24
      },
25
      "scoreResults": [
26
        {
27
          "name": "exact_match",
28
          "value": 0,
29
          "reason": "Exact match: No match"
30
        }
31
      ]
32
    }
33
  ],
34
  "resultUrl": "https://comet.com/opik/api/v1/session/redirect/experiments/?experiment_id=01975908-818f-765a-abv6-08d179c15610&dataset_id=01975908-810c-7663-b7a3-e3ae94484ca9&path=aHR0cHM6Ly9kZXYuY29tZXQuY29tL29dfWsvYXBp"
35
}