Manually logging experiments

1
import { Opik } from "opik";
2
3
const client = new Opik({
4
  apiKey: "your-api-key",
5
  apiUrl: "https://www.comet.com/opik/api",
6
  projectName: "your-project-name",
7
  workspaceName: "your-workspace-name",
8
});
9
const dataset = await client.getOrCreateDataset("My dataset");
10
11
await dataset.insert([
12
  {
13
    user_question: "What is the capital of France?",
14
    expected_output: "Paris"
15
  },
16
  {
17
    user_question: "What is the capital of Japan?",
18
    expected_output: "Tokyo"
19
  },
20
  {
21
    user_question: "What is the capital of Brazil?",
22
    expected_output: "Brasília"
23
  }
24
]);

1
const datasetItems = await dataset.getItems();
2
3
const mockResponses = {
4
  "What is the capital of France?": "The capital of France is Paris.",
5
  "What is the capital of Japan?": "Japan's capital is Tokyo.",
6
  "What is the capital of Brazil?": "The capital of Brazil is Rio de Janeiro."
7
}
8
9
// This would be replaced by your specific logic, the goal is simply to have an array of
10
// evaluation items with a dataset_item_id, evaluate_task_result and feedback_scores
11
const evaluationItems = datasetItems.map(item => {  
12
  const response = mockResponses[item.user_question] || "I don't know";
13
    return {
14
      dataset_item_id: item.id,
15
      evaluate_task_result: { prediction: response },
16
      feedback_scores: [{ name: "accuracy", value: response.includes(item.expected_output) ? 1.0 : 0.0, source: "sdk" }]
17
    }
18
});

1
import { Opik } from "opik";
2
3
const client = new Opik({
4
  apiKey: "your-api-key",
5
  apiUrl: "https://www.comet.com/opik/api",
6
  projectName: "your-project-name",
7
  workspaceName: "your-workspace-name",
8
});
9
10
const experimentName = "Bulk experiment upload";
11
const datasetName = "geography-questions";
12
const items = [
13
  {
14
    dataset_item_id: "dataset-item-id-1",
15
    evaluate_task_result: { prediction: "The capital of France is Paris." },
16
    feedback_scores: [{ name: "accuracy", value: 1.0, source: "sdk" }]
17
  }
18
];
19
20
await client.api.experiments.experimentItemsBulk({ experimentName, datasetName, items });

1
import { generateId } from "opik";
2
3
const experimentId = generateId();
4
const experimentName = "Bulk experiment upload";
5
// Split evaluation_items into two batches
6
const mid = Math.floor(evaluationItems.length / 2);
7
8
const halves = [
9
  evaluationItems.slice(0, mid),
10
  evaluationItems.slice(mid)
11
];
12
13
for (const half of halves) {
14
  await client.restClient.experiments.experimentItemsBulk({
15
    experimentId: experimentId,
16
    experimentName: experimentName,
17
    datasetName: "geography-questions",
18
    items: half.map(item => ({
19
      datasetItemId: item.datasetItemId,
20
      evaluateTaskResult: item.evaluateTaskResult,
21
      feedbackScores: item.feedbackScores.map(score => ({
22
        ...score,
23
        source: "sdk"
24
      }))
25
    }))
26
  });
27
}

1
import { Opik } from "opik";
2
3
// Configure Opik
4
const client = new Opik({
5
apiKey: "your-api-key",
6
apiUrl: "https://www.comet.com/opik/api",
7
projectName: "your-project-name",
8
workspaceName: "your-workspace-name",
9
});
10
11
// Step 1: Create dataset
12
const dataset = await client.getOrCreateDataset("geography-questions");
13
14
const localDatasetItems = [
15
{
16
  user_question: "What is the capital of France?",
17
  expected_output: "Paris"
18
},
19
{
20
  user_question: "What is the capital of Japan?",
21
  expected_output: "Tokyo"
22
}
23
];
24
25
await dataset.insert(localDatasetItems);
26
27
// Step 2: Get dataset items and prepare evaluation results
28
const datasetItems = await dataset.getItems();
29
30
// Helper function to get dataset item ID
31
const getDatasetItem = (country: string) => {
32
return datasetItems.find(item =>
33
  item.user_question.toLowerCase().includes(country.toLowerCase())
34
);
35
};
36
37
// Prepare evaluation results
38
const evaluationItems = [
39
{
40
  dataset_item_id: getDatasetItem("France")?.id,
41
  evaluate_task_result: { prediction: "The capital of France is Paris." },
42
  feedback_scores: [{ name: "accuracy", value: 1.0 }]
43
},
44
{
45
  dataset_item_id: getDatasetItem("Japan")?.id,
46
  evaluate_task_result: { prediction: "Japan's capital is Tokyo." },
47
  feedback_scores: [{ name: "accuracy", value: 1.0 }]
48
}
49
];
50
51
// Step 3: Log experiment results
52
const experimentName = `geography-bot-${Math.random().toString(36).substr(2, 4)}`;
53
await client.api.experiments.experimentItemsBulk({
54
experimentName,
55
datasetName: "geography-questions",
56
items: evaluationItems.map(item => ({
57
  datasetItemId: item.dataset_item_id,
58
  evaluateTaskResult: item.evaluate_task_result,
59
  feedbackScores: item.feedback_scores.map(score => ({
60
    ...score,
61
    source: "sdk"
62
  }))
63
}))
64
});
65
66
console.log(`Experiment '${experimentName}' created successfully!`);