{"id":11840,"date":"2024-10-24T13:36:02","date_gmt":"2024-10-24T21:36:02","guid":{"rendered":"https:\/\/live-cometml.pantheonsite.io\/?page_id=11840"},"modified":"2026-05-06T18:36:55","modified_gmt":"2026-05-06T18:36:55","slug":"llm-course","status":"publish","type":"page","link":"https:\/\/www.comet.com\/site\/llm-course\/","title":{"rendered":"LLM Course"},"content":{"rendered":"\n<div class=\"wp-block-group is-layout-constrained wp-container-core-group-is-layout-960ecac0 wp-block-group-is-layout-constrained\">\n<div class=\"wp-block-columns alignwide is-layout-flex wp-container-core-columns-is-layout-5ee75b69 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-layout-flow wp-container-core-column-is-layout-bb004fa3 wp-block-column-is-layout-flow\" style=\"flex-basis:33.33%\">\n<p class=\"wp-embed-aspect-16-9 wp-has-aspect-ratio has-tertiary-color has-text-color has-link-color has-display-xxs-font-size wp-elements-48129670fc9dd3466d5ccc30c4c1365e wp-block-paragraph\" style=\"font-style:normal;font-weight:600\">FREE COURSE<\/p>\n\n\n\n<h1 class=\"wp-block-heading wp-embed-aspect-16-9 wp-has-aspect-ratio unstyled-links has-display-s-font-size\"><a href=\"https:\/\/www.comet.com\/site\/blog\/llm-evaluation-guide\/\">LLM Evaluation<\/a> with Opik<\/h1>\n\n\n\n<p class=\"wp-embed-aspect-16-9 wp-has-aspect-ratio unstyled-links wp-block-paragraph\">Learn to test and evaluate your LLM applications using the latest tools and techniques, including <a href=\"https:\/\/www.comet.com\/site\/blog\/llm-as-a-judge\/\">LLM-as-a-judge<\/a> metrics and production <a href=\"https:\/\/www.comet.com\/site\/blog\/llm-monitoring\/\">LLM monitoring<\/a>.<br><br><strong>Registration for this course is now closed<\/strong>, but the LLM observability and evaluation concepts covered on this page are as critical as ever to develop and ship <a href=\"https:\/\/www.comet.com\/site\/blog\/ai-agents\/\">AI agents<\/a> that actually work. Discover essential LLM eval workflows right inside Opik and in our documentation.<\/p>\n\n\n\n<div class=\"wp-block-buttons wp-embed-aspect-16-9 wp-has-aspect-ratio is-layout-flex wp-block-buttons-is-layout-flex\">\n<div class=\"wp-block-button\"><a class=\"wp-block-button__link has-background-color has-tertiary-background-color has-text-color has-background has-link-color wp-element-button\" href=\"https:\/\/www.comet.com\/site\/products\/opik\/\">Explore Opik<\/a><\/div>\n<\/div>\n\n\n\n<p class=\"wp-embed-aspect-16-9 wp-has-aspect-ratio has-tertiary-color has-text-color has-link-color has-display-xxs-font-size wp-elements-cf894550ed3d3dbfe89807302f53a77b wp-block-paragraph\" style=\"margin-top:var(--wp--preset--spacing--40);font-style:normal;font-weight:600\">IN COLLABORATION WITH<\/p>\n\n\n\n<div class=\"wp-block-group is-nowrap is-layout-flex wp-container-core-group-is-layout-f56f613f wp-block-group-is-layout-flex\">\n<figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"293\" height=\"77\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/dair2.png\" alt=\"\" class=\"wp-image-11842\" style=\"object-fit:cover\"\/><\/figure>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:66.66%\">\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"LLM Evaluations\" width=\"500\" height=\"281\" src=\"https:\/\/www.youtube.com\/embed\/BQIL3VXu0XA?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen><\/iframe>\n<\/div><\/figure>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-group alignwide has-custom-background-2-background-color has-background is-layout-constrained wp-container-core-group-is-layout-1eb967e0 wp-block-group-is-layout-constrained\" style=\"border-radius:30px;padding-top:var(--wp--preset--spacing--16);padding-right:var(--wp--preset--spacing--50);padding-bottom:var(--wp--preset--spacing--16);padding-left:var(--wp--preset--spacing--50)\">\n<div class=\"wp-block-columns alignwide is-layout-flex wp-container-core-columns-is-layout-f56f613f wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-layout-flow wp-container-core-column-is-layout-0e604a4a wp-block-column-is-layout-flow\">\n<div class=\"wp-block-group is-nowrap is-layout-flex wp-container-core-group-is-layout-28607b0a wp-block-group-is-layout-flex\">\n<div class=\"wp-block-group has-body-m-font-size is-layout-constrained wp-block-group-is-layout-constrained\" style=\"line-height:1.1\">\n<figure class=\"wp-block-image size-large is-resized\"><img decoding=\"async\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2025\/05\/circle-user-regular.svg\" alt=\"\" class=\"wp-image-16086\" style=\"object-fit:contain;width:1em;height:1em\"\/><\/figure>\n<\/div>\n\n\n\n<div class=\"wp-block-group is-layout-constrained wp-container-core-group-is-layout-8f97d2ea wp-block-group-is-layout-constrained\">\n<h2 class=\"wp-block-heading has-tertiary-color has-text-color has-link-color has-body-m-font-size wp-elements-ab2baecf11006baf87dc71a996ecc11f\">Expert Instructors<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Taught by industry leaders<\/p>\n<\/div>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-layout-flow wp-container-core-column-is-layout-0e604a4a wp-block-column-is-layout-flow\">\n<div class=\"wp-block-group is-nowrap is-layout-flex wp-container-core-group-is-layout-28607b0a wp-block-group-is-layout-flex\">\n<div class=\"wp-block-group has-body-m-font-size is-layout-constrained wp-block-group-is-layout-constrained\" style=\"line-height:1.1\">\n<figure class=\"wp-block-image size-large is-resized\"><img decoding=\"async\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2025\/05\/calendar-check-regular.svg\" alt=\"\" class=\"wp-image-16088\" style=\"object-fit:contain;width:1em;height:1em\"\/><\/figure>\n<\/div>\n\n\n\n<div class=\"wp-block-group is-layout-constrained wp-container-core-group-is-layout-8f97d2ea wp-block-group-is-layout-constrained\">\n<h2 class=\"wp-block-heading has-tertiary-color has-text-color has-link-color has-body-m-font-size wp-elements-ba2c6211a54ede7bf95193f86033a733\">Self-Paced<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Learn at your own speed<\/p>\n<\/div>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-layout-flow wp-container-core-column-is-layout-0e604a4a wp-block-column-is-layout-flow\">\n<div class=\"wp-block-group is-nowrap is-layout-flex wp-container-core-group-is-layout-28607b0a wp-block-group-is-layout-flex\">\n<div class=\"wp-block-group has-body-m-font-size is-layout-constrained wp-block-group-is-layout-constrained\" style=\"line-height:1.1\">\n<figure class=\"wp-block-image size-large is-resized\"><img decoding=\"async\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2025\/05\/fire-flame-curved-solid.svg\" alt=\"\" class=\"wp-image-16089\" style=\"object-fit:contain;width:1em;height:1em\"\/><\/figure>\n<\/div>\n\n\n\n<div class=\"wp-block-group is-layout-constrained wp-container-core-group-is-layout-8f97d2ea wp-block-group-is-layout-constrained\">\n<h2 class=\"wp-block-heading has-tertiary-color has-text-color has-link-color has-body-m-font-size wp-elements-8396e2773e4dd5afd0d4d5ccff92f57d\">Built with Opik<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Use open source tools and models<\/p>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-group alignwide is-content-justification-space-between is-nowrap is-layout-flex wp-container-core-group-is-layout-73d85c4d wp-block-group-is-layout-flex\">\n<figure class=\"wp-block-image size-large wp-container-content-b04ba3db\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"272\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/hf-logo-with-title-1024x272.png\" alt=\"Hugging Face logo\" class=\"wp-image-7894\" style=\"object-fit:cover\" srcset=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/hf-logo-with-title-1024x272.png 1024w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/hf-logo-with-title-300x80.png 300w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/hf-logo-with-title-768x204.png 768w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/hf-logo-with-title-1536x408.png 1536w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/hf-logo-with-title.png 1926w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-large wp-container-content-b04ba3db\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"348\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Logo-Final-1024x348.png\" alt=\"opik logo\" class=\"wp-image-11803\" srcset=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Logo-Final-1024x348.png 1024w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Logo-Final-300x102.png 300w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Logo-Final-768x261.png 768w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Logo-Final-1536x522.png 1536w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Logo-Final.png 1551w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"182\" height=\"31\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Untitled.png\" alt=\"LiteLLM logo\" class=\"wp-image-11804\"\/><\/figure>\n<\/div>\n\n\n\n<div class=\"wp-block-columns alignwide is-layout-flex wp-container-core-columns-is-layout-800df3e3 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\">\n<h2 class=\"wp-block-heading has-display-s-font-size\" style=\"font-style:normal;font-weight:400\">Course Description<\/h2>\n\n\n\n<ul class=\"wp-block-list checkmark-list\">\n<li>Level: Beginner<\/li>\n\n\n\n<li>Duration: 1 Hour<\/li>\n\n\n\n<li>Audience: Data Scientists\/Software Engineers<\/li>\n\n\n\n<li>Prerequisites: Basic ML knowledge, Python experience<\/li>\n<\/ul>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\">\n<h2 class=\"wp-block-heading has-display-s-font-size\" style=\"font-style:normal;font-weight:400\">Who Is This For?<\/h2>\n\n\n\n<ul class=\"wp-block-list checkmark-list\">\n<li>AI Developers<\/li>\n\n\n\n<li>Anyone curious about LLMs<\/li>\n\n\n\n<li>Engineers<\/li>\n\n\n\n<li>Data Scientists<\/li>\n<\/ul>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-columns alignwide is-layout-flex wp-container-core-columns-is-layout-800df3e3 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\">\n<h2 class=\"wp-block-heading has-display-s-font-size\" style=\"font-style:normal;font-weight:400\">Why This Course?<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">This is the only course completely focused on applying state-of-the-art LLM evaluation techniques to real world applications. We will cover some theory, but this is first and foremost a course in applied AI, not mathematics.<\/p>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\">\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"513\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Screen-Shot-2024-10-24-at-5.46.17-PM-1024x513.png\" alt=\"\" class=\"wp-image-11851\" srcset=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Screen-Shot-2024-10-24-at-5.46.17-PM-1024x513.png 1024w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Screen-Shot-2024-10-24-at-5.46.17-PM-300x150.png 300w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Screen-Shot-2024-10-24-at-5.46.17-PM-768x385.png 768w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/Screen-Shot-2024-10-24-at-5.46.17-PM.png 1526w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-columns alignwide are-vertically-aligned-center is-layout-flex wp-container-core-columns-is-layout-f56f613f wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-center is-layout-flow wp-block-column-is-layout-flow\">\n<div class=\"wp-block-group is-nowrap is-layout-flex wp-container-core-group-is-layout-f56f613f wp-block-group-is-layout-flex\">\n<figure class=\"wp-block-image size-full is-resized is-style-rounded\"><img loading=\"lazy\" decoding=\"async\" width=\"800\" height=\"800\" src=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/elvis-circular-headshot.png\" alt=\"elvis headshot\" class=\"wp-image-7822\" style=\"box-shadow:var(--wp--preset--shadow--light);width:300px\" srcset=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/elvis-circular-headshot.png 800w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/elvis-circular-headshot-300x300.png 300w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/elvis-circular-headshot-150x150.png 150w, https:\/\/www.comet.com\/site\/wp-content\/uploads\/2023\/10\/elvis-circular-headshot-768x768.png 768w\" sizes=\"auto, (max-width: 800px) 100vw, 800px\" \/><\/figure>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-center is-layout-flow wp-block-column-is-layout-flow\">\n<h2 class=\"wp-block-heading has-display-s-font-size\" style=\"font-style:normal;font-weight:400\">Taught By An Expert<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Elvis is the co-founder of DAIR.AI, where he leads all AI research, education, and engineering efforts. He focuses on training and building large language models (LLMs) and information retrieval systems. Previous to this, he was at Meta AI where he supported and advised world-class products and teams such as FAIR, PyTorch, and Papers with Code. He was also previously an education architect at Elastic where he developed technical curriculum and courses for the Elastic Stack. <\/p>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-buttons is-content-justification-center is-layout-flex wp-block-buttons-is-layout-flex\">\n<div class=\"wp-block-button taught-experts\"><a class=\"wp-block-button__link has-background-color has-tertiary-background-color has-text-color has-background has-link-color wp-element-button\" href=\"https:\/\/www.comet.com\/site\/products\/opik\/\">Explore Opik<\/a><\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-group alignfull has-background-color has-custom-background-dark-background-color has-text-color has-background has-link-color wp-elements-9a3cd0a0792d528a0230a00b17f6ca24 is-layout-constrained wp-block-group-is-layout-constrained\" style=\"padding-top:var(--wp--preset--spacing--100);padding-bottom:var(--wp--preset--spacing--100)\">\n<h2 class=\"wp-block-heading has-text-align-center has-text-color\" style=\"color:#ffffff;margin-top:0;margin-bottom:var(--wp--preset--spacing--40)\">What You&#8217;ll Learn<\/h2>\n\n\n\n<div class=\"wp-block-group has-tertiary-background-color has-background is-layout-constrained wp-container-core-group-is-layout-29844009 wp-block-group-is-layout-constrained\" style=\"border-radius:8px;padding-top:var(--wp--preset--spacing--40);padding-right:var(--wp--preset--spacing--40);padding-bottom:var(--wp--preset--spacing--40);padding-left:var(--wp--preset--spacing--40)\">\n<div class=\"wp-block-group is-vertical is-layout-flex wp-container-core-group-is-layout-c24fd9a3 wp-block-group-is-layout-flex\">\n<h3 class=\"wp-block-heading\">Syllabus<\/h3>\n\n\n\n<p class=\"has-secondary-color has-text-color has-link-color wp-elements-9222865b78974c3cc718c780690e9aea wp-block-paragraph\">1 Course &#8211; 8 Lessons &#8211; 1 Project<\/p>\n<\/div>\n\n\n\n<div class=\"wp-block-comet-accordion-accordion comet-accordion\" data-allow-multiple=\"false\">\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>1 &#8211; Introduction to LLM Evaluation<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<ul style=\"color:#ffffff\" class=\"wp-block-list checkmark-list__purple has-text-color\">\n<li>Brief introduction to LLM evaluations<\/li>\n\n\n\n<li>Explore the challenges of evaluating LLM applications<\/li>\n\n\n\n<li>Overview some common use cases<\/li>\n<\/ul>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>2 &#8211; Tools &amp; Frameworks for LLM Evaluation<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<ul style=\"color:#ffffff\" class=\"wp-block-list checkmark-list__purple has-text-color\">\n<li>Explore the LLM evaluation ecosystem<\/li>\n\n\n\n<li>Familiarize yourself with Opik<\/li>\n\n\n\n<li>Stand up your first simple evaluation suite<\/li>\n<\/ul>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>3 &#8211; Project: Evaluating a Chatbot<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<ul style=\"color:#ffffff\" class=\"wp-block-list checkmark-list__purple has-text-color\">\n<li>Kickoff the main section of the course with a real project<\/li>\n\n\n\n<li>Learn about common chatbot architectures<\/li>\n\n\n\n<li>Implement evaluations for a real LLM application<\/li>\n<\/ul>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>4 &#8211; Building an LLM Evaluation Pipeline<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<ul style=\"color:#ffffff\" class=\"wp-block-list checkmark-list__purple has-text-color\">\n<li>Go from simple evaluations to robust evaluation pipelines<\/li>\n\n\n\n<li>Explore common workflows for production evaluation systems<\/li>\n\n\n\n<li>Get your feet wet with manual evaluations<\/li>\n<\/ul>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>5 &#8211; Heuristic Metrics for LLM Evaluation<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<ul style=\"color:#ffffff\" class=\"wp-block-list checkmark-list__purple has-text-color\">\n<li>Familiarize yourself with some &#8220;classic&#8221; metrics<\/li>\n\n\n\n<li>Implement heuristic metrics from scratch<\/li>\n\n\n\n<li>Understand the benefits and challenges of heuristic evaluations<\/li>\n<\/ul>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>6 &#8211; LLM-Based Metrics for LLM Evaluation<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<ul style=\"color:#ffffff\" class=\"wp-block-list checkmark-list__purple has-text-color\">\n<li>Learn about LLM-as-a-judge metrics<\/li>\n\n\n\n<li>Implement custom LLM-based metrics from scratch<\/li>\n\n\n\n<li>Learn to test for hallucinations, factuality, and more<\/li>\n<\/ul>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>7 &#8211; Testing &amp; Monitoring LLM Applications<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<ul style=\"color:#ffffff\" class=\"wp-block-list checkmark-list__purple has-text-color\">\n<li>Learn to monitor deployed LLM applications<\/li>\n\n\n\n<li>Implement LLM unit tests with PyTest and Opik<\/li>\n\n\n\n<li>Understand the role of observability in LLM applications<\/li>\n<\/ul>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>8 &#8211; The Future of LLM Evaluation<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<ul style=\"color:#ffffff\" class=\"wp-block-list checkmark-list__purple has-text-color\">\n<li>Explore advanced techniques for LLM evaluation<\/li>\n\n\n\n<li>Understand safety evaluations and responsible AI<\/li>\n\n\n\n<li>Overview the next steps in your learning journey<\/li>\n<\/ul>\n<\/div><\/details>\n<\/div>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-group is-layout-constrained wp-container-core-group-is-layout-3e84bdd8 wp-block-group-is-layout-constrained\">\n<h2 class=\"wp-block-heading has-text-align-center has-display-s-font-size\" style=\"font-style:normal;font-weight:400\">Frequently Asked Questions<\/h2>\n\n\n\n<div class=\"wp-block-comet-accordion-accordion comet-accordion\" data-allow-multiple=\"false\">\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>What are the prerequisites for this course?<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<p class=\"wp-block-paragraph\">This course assumes no advanced math background. We will not be diving deep into the theory behind LLMs. All you need to get started is some basic proficiency in Python and a general understanding of deep learning. <\/p>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>Will it cost me anything?<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<p class=\"wp-block-paragraph\">The course content is 100% free. Every lesson can be completed using completely free and open source models via LiteLLM and Opik. You can also use the OpenAI or Anthropic API, if you prefer. <\/p>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>How much time should I commit?<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<p class=\"wp-block-paragraph\">The course is self-paced, so you can spend as little or as much time as you want. That said, students who set aside a meaningful block of time each week\u2014whatever &#8220;meaningful&#8221; means for your schedule\u2014tend to see the best results. <\/p>\n<\/div><\/details>\n\n\n\n<details class=\"wp-block-comet-accordion-item comet-accordion__item\"><summary class=\"comet-accordion__item-summary\"><span>How long will this course take?<\/span><span class=\"comet-accordion__item-icon\" aria-hidden=\"true\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"16\" height=\"16\" fill=\"none\" stroke=\"#191A1C\"><path stroke-linecap=\"round\" stroke-linejoin=\"round\" stroke-width=\"2\" d=\"M8 1v14m7-7H1\"><\/path><\/svg><\/span><\/summary><div class=\"comet-accordion__item-content\">\n<p class=\"wp-block-paragraph\">Your time to completion will vary depending on how much time you have available. In general, we recommend one week per module as a realistic pace for most people, meaning the course would take six weeks total. Of course, you can take as long as you&#8217;d like. <\/p>\n<\/div><\/details>\n<\/div>\n<\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-group alignfull is-layout-constrained wp-block-group-is-layout-constrained\" style=\"margin-top:0;margin-bottom:0;padding-top:var(--wp--preset--spacing--100);padding-bottom:var(--wp--preset--spacing--100)\">\n<div class=\"wp-block-group alignwide has-background-color has-accent-background-color has-text-color has-background has-link-color wp-elements-913b3d1175baaf6000b981ff2671022c is-layout-constrained wp-container-core-group-is-layout-fb75a317 wp-block-group-is-layout-constrained\" style=\"border-radius:12px;padding-top:var(--wp--preset--spacing--50);padding-right:var(--wp--preset--spacing--50);padding-bottom:var(--wp--preset--spacing--50);padding-left:var(--wp--preset--spacing--50);background-image:url(&apos;https:\/\/www.comet.com\/site\/wp-content\/uploads\/2025\/03\/pattern-4.png&apos;);background-position:50% 50%;background-size:cover;background-attachment:scroll;\">\n<h2 class=\"wp-block-heading has-text-align-center has-background-color has-text-color has-link-color wp-elements-c2bd0302519f120fb6fc60dfafbc0b43\" id=\"h-get-started-today-for-free\" style=\"margin-top:var(--wp--preset--spacing--24)\">Try Opik Free<\/h2>\n\n\n\n<p class=\"has-text-align-center has-body-m-font-size wp-block-paragraph\">Opik is free to use for as long as you like, with generous monthly usage limits and everything you need to start logging, evaluating, and improving your agent&#8217;s behavior included.<\/p>\n\n\n\n<div class=\"wp-block-buttons is-content-justification-center is-layout-flex wp-container-core-buttons-is-layout-43139366 wp-block-buttons-is-layout-flex\" style=\"margin-bottom:var(--wp--preset--spacing--24)\">\n<div class=\"wp-block-button\"><a rel=\"\" target=\"\" class=\"wp-block-button__link has-tertiary-background-color has-background wp-element-button\" href=\"https:\/\/www.comet.com\/signup?from=llm\">Create Free Account<\/a><\/div>\n\n\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>FREE COURSE LLM Evaluation with Opik Learn to test and evaluate your LLM applications using the latest tools and techniques, including LLM-as-a-judge metrics and production LLM monitoring. Registration for this course is now closed, but the LLM observability and evaluation concepts covered on this page are as critical as ever to develop and ship AI [&hellip;]<\/p>\n","protected":false},"author":25,"featured_media":0,"parent":0,"menu_order":0,"comment_status":"closed","ping_status":"closed","template":"","meta":{"customer_name":"","customer_description":"","customer_industry":"","customer_technologies":"","customer_logo":"","footnotes":""},"coauthors":[142],"class_list":["post-11840","page","type-page","status-publish","hentry"],"yoast_head":"<!-- This site is optimized with the Yoast SEO Premium plugin v25.9 (Yoast SEO v25.9) - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>LLMOps Course: Building with LLMs<\/title>\n<meta name=\"description\" content=\"Register for this course and learn to build modern software with LLMs using the newest tools and techniques in the field.\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.comet.com\/site\/llm-course\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"LLM Course\" \/>\n<meta property=\"og:description\" content=\"Register for this course and learn to build modern software with LLMs using the newest tools and techniques in the field.\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.comet.com\/site\/llm-course\/\" \/>\n<meta property=\"og:site_name\" content=\"Comet\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/cometdotml\" \/>\n<meta property=\"article:modified_time\" content=\"2026-05-06T18:36:55+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/dair2.png\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:site\" content=\"@Cometml\" \/>\n<meta name=\"twitter:label1\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data1\" content=\"5 minutes\" \/>\n\t<meta name=\"twitter:label2\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data2\" content=\"Caleb Kaiser\" \/>\n<!-- \/ Yoast SEO Premium plugin. -->","yoast_head_json":{"title":"LLMOps Course: Building with LLMs","description":"Register for this course and learn to build modern software with LLMs using the newest tools and techniques in the field.","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.comet.com\/site\/llm-course\/","og_locale":"en_US","og_type":"article","og_title":"LLM Course","og_description":"Register for this course and learn to build modern software with LLMs using the newest tools and techniques in the field.","og_url":"https:\/\/www.comet.com\/site\/llm-course\/","og_site_name":"Comet","article_publisher":"https:\/\/www.facebook.com\/cometdotml","article_modified_time":"2026-05-06T18:36:55+00:00","og_image":[{"url":"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/dair2.png","type":"","width":"","height":""}],"twitter_card":"summary_large_image","twitter_site":"@Cometml","twitter_misc":{"Est. reading time":"5 minutes","Written by":"Caleb Kaiser"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.comet.com\/site\/llm-course\/","url":"https:\/\/www.comet.com\/site\/llm-course\/","name":"LLMOps Course: Building with LLMs","isPartOf":{"@id":"https:\/\/www.comet.com\/site\/#website"},"primaryImageOfPage":{"@id":"https:\/\/www.comet.com\/site\/llm-course\/#primaryimage"},"image":{"@id":"https:\/\/www.comet.com\/site\/llm-course\/#primaryimage"},"thumbnailUrl":"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/dair2.png","datePublished":"2024-10-24T21:36:02+00:00","dateModified":"2026-05-06T18:36:55+00:00","description":"Register for this course and learn to build modern software with LLMs using the newest tools and techniques in the field.","breadcrumb":{"@id":"https:\/\/www.comet.com\/site\/llm-course\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.comet.com\/site\/llm-course\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.comet.com\/site\/llm-course\/#primaryimage","url":"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/dair2.png","contentUrl":"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2024\/10\/dair2.png","width":293,"height":77},{"@type":"BreadcrumbList","@id":"https:\/\/www.comet.com\/site\/llm-course\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/www.comet.com\/site\/"},{"@type":"ListItem","position":2,"name":"LLM Course"}]},{"@type":"WebSite","@id":"https:\/\/www.comet.com\/site\/#website","url":"https:\/\/www.comet.com\/site\/","name":"Comet","description":"Build Better Models Faster","publisher":{"@id":"https:\/\/www.comet.com\/site\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.comet.com\/site\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/www.comet.com\/site\/#organization","name":"Comet ML, Inc.","alternateName":"Comet","url":"https:\/\/www.comet.com\/site\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.comet.com\/site\/#\/schema\/logo\/image\/","url":"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2025\/01\/logo_comet_square.png","contentUrl":"https:\/\/www.comet.com\/site\/wp-content\/uploads\/2025\/01\/logo_comet_square.png","width":310,"height":310,"caption":"Comet ML, Inc."},"image":{"@id":"https:\/\/www.comet.com\/site\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/cometdotml","https:\/\/x.com\/Cometml","https:\/\/www.youtube.com\/channel\/UCmN63HKvfXSCS-UwVwmK8Hw"]}]}},"jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/pages\/11840","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/pages"}],"about":[{"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/types\/page"}],"author":[{"embeddable":true,"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/users\/25"}],"replies":[{"embeddable":true,"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/comments?post=11840"}],"version-history":[{"count":2,"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/pages\/11840\/revisions"}],"predecessor-version":[{"id":19945,"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/pages\/11840\/revisions\/19945"}],"wp:attachment":[{"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/media?parent=11840"}],"wp:term":[{"taxonomy":"author","embeddable":true,"href":"https:\/\/www.comet.com\/site\/wp-json\/wp\/v2\/coauthors?post=11840"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}