+ The academy page you're looking for doesn't exist or has moved.
+
+
+ Back to Academy
+
+
+ );
+}
diff --git a/app/academy/[[...slug]]/page.tsx b/app/academy/[[...slug]]/page.tsx
new file mode 100644
index 000000000..0d6cb1bf8
--- /dev/null
+++ b/app/academy/[[...slug]]/page.tsx
@@ -0,0 +1,71 @@
+import type { Metadata } from "next";
+import { academySource } from "@/lib/source";
+import { buildOgImageUrl, buildPageUrl } from "@/lib/og-url";
+import { DocsPage } from "fumadocs-ui/page";
+import { notFound } from "next/navigation";
+import { DocsContributors } from "@/components/DocsContributors";
+import { DocBodyChrome } from "@/components/DocBodyChrome";
+import { getMDXComponents } from "@/mdx-components";
+import type { ComponentType } from "react";
+
+type PageProps = {
+ params: Promise<{ slug?: string[] }>;
+};
+
+export default async function AcademyPage(props: PageProps) {
+ const params = await props.params;
+ const slug = params.slug ?? [];
+ const page = academySource.getPage(slug);
+
+ if (!page) notFound();
+
+ const { toc } = page.data;
+ const MDX = page.data.body as ComponentType<{ components?: Record }>;
+
+ return (
+ }}
+ >
+
+
+
+
+ );
+}
+
+export async function generateMetadata(props: PageProps): Promise {
+ const params = await props.params;
+ const slug = params.slug ?? [];
+ const page = academySource.getPage(slug);
+ if (!page)
+ return {
+ title: "Not Found",
+ };
+ const pageData = page.data as typeof page.data & {
+ canonical?: string | null;
+ seoTitle?: string | null;
+ ogImage?: string | null;
+ };
+ const pagePath = `/academy${slug.length > 0 ? `/${slug.join("/")}` : ""}`;
+ const canonicalUrl = pageData.canonical ?? buildPageUrl(pagePath);
+ const seoTitle = pageData.seoTitle || page.data.title;
+ const ogImage = buildOgImageUrl({
+ title: seoTitle,
+ description: page.data.description,
+ section: "Academy",
+ staticOgImage: pageData.ogImage,
+ });
+ return {
+ title: seoTitle,
+ description: page.data.description ?? undefined,
+ alternates: { canonical: canonicalUrl },
+ openGraph: { images: [{ url: ogImage }], url: canonicalUrl },
+ twitter: { images: [{ url: ogImage }] },
+ };
+}
+
+export function generateStaticParams() {
+ return academySource.generateParams();
+}
diff --git a/app/academy/layout.tsx b/app/academy/layout.tsx
new file mode 100644
index 000000000..f24fdaceb
--- /dev/null
+++ b/app/academy/layout.tsx
@@ -0,0 +1,10 @@
+import { academySource, getPageTreeWithShortTitles } from "@/lib/source";
+import { SharedDocsLayout } from "@/app/docs/SharedDocsLayout";
+
+export default function AcademyLayout({
+ children,
+}: {
+ children: React.ReactNode;
+}) {
+ return {children};
+}
diff --git a/components/NavLinks.tsx b/components/NavLinks.tsx
index 5624a7eb7..fceaa0117 100644
--- a/components/NavLinks.tsx
+++ b/components/NavLinks.tsx
@@ -22,6 +22,7 @@ const productLinks = [
];
const resourcesLinks = [
+ { name: "Academy", href: "/academy" },
{ name: "Blog", href: "/blog" },
{ name: "Changelog", href: "/changelog" },
{ name: "Roadmap", href: "/docs/roadmap" },
diff --git a/content/academy/building-datasets.mdx b/content/academy/building-datasets.mdx
new file mode 100644
index 000000000..aca1bd940
--- /dev/null
+++ b/content/academy/building-datasets.mdx
@@ -0,0 +1,310 @@
+---
+title: Building Datasets
+description: "Learn how to test LLM applications with automated evaluation, datasets, and experiment runners."
+---
+
+# Building a Good Dataset
+
+esting LLM applications presents unique challenges. Unlike traditional software where outputs are deterministic, LLMs produce varied responses that can't be verified with simple equality checks. Yet the need for systematic testing remains critical—how do you ensure your AI application works reliably across deployments?
+
+This guide shows you how to implement automated tests for LLM applications using datasets and experiment runners, inspired by [Hamel Husain's testing framework](https://hamel.dev/blog/posts/evals/#level-1-unit-tests).
+
+## Testing vs. Evaluation in LLM Applications
+
+Before diving in, let's clarify some terminology that often causes confusion:
+
+**Testing** typically means running automated checks that produce pass/fail results. You write assertions like `assert result == expected` and your test suite tells you if something broke.
+
+**Evaluation** is about measuring quality. How accurate is your model? How helpful are the responses? These are scored on continuous scales rather than binary pass/fail.
+
+Traditional testing relies on predictable outputs. You assert that `add(2, 3)` returns `5`. But when you ask an LLM "What is the capital of France?", you might get "Paris", "The capital is Paris", "Paris, France", or a longer explanation. All are correct, but none match exactly.
+
+This variability doesn't mean we can't test LLM applications—it means we need different testing strategies. In LLM applications, these concepts blend together. You "test" your application by "evaluating" its outputs with scoring functions. A test passes if the evaluation score meets your threshold.
+
+### Unit Tests vs What We're Building
+
+Hamel Husain calls this approach "Level 1: Unit Tests" in his framework, and we'll use similar terminology. However, it's worth noting that these aren't traditional unit tests:
+
+**Traditional unit tests**:
+- Test isolated code units
+- Deterministic (same input = same output)
+- Fast, no external dependencies
+
+**LLM application tests**:
+- Test application behavior
+- Non-deterministic (same input can produce different outputs)
+- Slower execution
+
+Think of these as **automated regression tests** that verify your LLM application maintains acceptable quality as you make changes. The "unit" being tested is your application's behavior on specific inputs.
+
+## The Solution: Datasets + Experiment Runners + Evaluators
+
+The approach combines three components:
+
+1. **Datasets**: Collections of input/output pairs that represent your test cases
+2. **Experiment Runners**: Execute your LLM application against the dataset
+3. **Evaluators**: Score the outputs programmatically instead of checking exact matches
+
+Let's see how this works in practice.
+
+## Example: Testing a Geography Question Answering System
+
+Here's a complete example testing an LLM application that answers geography questions, using Langfuse's Experiment Runner SDK and local test data.
+
+First, setup the environment variables:
+
+```sh
+# .env file
+OPENAI_API_KEY=your_openai_api_key
+LANGFUSE_PUBLIC_KEY=your_langfuse_public_key
+LANGFUSE_SECRET_KEY=your_langfuse_secret_key
+LANGFUSE_BASE_URL=https://cloud.langfuse.com
+```
+
+You can export the environment variables to your shell:
+
+```
+export $(grep -v '^#' .env)
+```
+
+Now create the test file:
+
+```python
+# test_geography_experiment.py
+import pytest
+from langfuse import get_client, Evaluation, Langfuse
+from langfuse.openai import OpenAI
+
+# Each test case includes both the input and expected output. The expected output
+# serves as a reference for evaluation, not as an exact string match.
+test_data = [
+ {"input": "What is the capital of France?", "expected_output": "Paris"},
+ {"input": "What is the capital of Germany?", "expected_output": "Berlin"},
+ {"input": "What is the capital of Spain?", "expected_output": "Madrid"},
+]
+
+# The task function wraps your LLM application logic. It receives each test item and
+# returns the LLM's response. This should be your full LLM application logic.
+def geography_task(*, item, **kwargs):
+ """Task function that answers geography questions"""
+ question = item["input"]
+ response = OpenAI().chat.completions.create(
+ model="gpt-4",
+ messages=[{"role": "user", "content": question}]
+ )
+ return response.choices[0].message.content
+
+# This evaluator checks if the expected answer appears anywhere in the output, accounting
+# for LLM verbosity. You could also use more sophisticated evaluators, including LLM-as-a-judge
+# for semantic similarity.
+def accuracy_evaluator(*, input, output, expected_output, **kwargs):
+ """Evaluator that checks if the expected answer is in the output"""
+ if expected_output and expected_output.lower() in output.lower():
+ return Evaluation(name="accuracy", value=1.0)
+
+ return Evaluation(name="accuracy", value=0.0)
+
+# Run-level evaluators aggregate scores across all test items, giving you a single
+# metric to assert against.
+def average_accuracy_evaluator(*, item_results, **kwargs):
+ """Run evaluator that calculates average accuracy across all items"""
+ accuracies = [
+ eval.value for result in item_results
+ for eval in result.evaluations if eval.name == "accuracy"
+ ]
+
+ if not accuracies:
+ return Evaluation(name="avg_accuracy", value=None)
+
+ avg = sum(accuracies) / len(accuracies)
+
+ return Evaluation(name="avg_accuracy", value=avg, comment=f"Average accuracy: {avg:.2%}")
+
+@pytest.fixture
+def langfuse_client() -> Langfuse:
+ """Initialize Langfuse client for testing"""
+ return get_client()
+
+def test_geography_accuracy_passes(langfuse_client: Langfuse):
+ """Test that passes when accuracy is above threshold"""
+ result = langfuse_client.run_experiment(
+ name="Geography Test - Should Pass",
+ data=test_data,
+ task=geography_task,
+ evaluators=[accuracy_evaluator],
+ run_evaluators=[average_accuracy_evaluator]
+ )
+
+ # Access the run evaluator result directly
+ avg_accuracy = next(
+ eval.value for eval in result.run_evaluations
+ if eval.name == "avg_accuracy"
+ )
+
+ # Assert minimum accuracy threshold
+ assert avg_accuracy >= 0.8, f"Average accuracy {avg_accuracy:.2f} below threshold 0.8"
+```
+
+
+ **Set Appropriate Thresholds**: Not all tests need 100% accuracy. Set realistic thresholds based on your application's requirements:
+
+ ```python
+ # Strict threshold for critical functionality
+ assert avg_accuracy >= 0.95, "Critical tests must have 95%+ accuracy"
+
+ # Relaxed threshold for experimental features
+ assert avg_accuracy >= 0.70, "Experimental feature needs improvement"
+ ```
+
+
+You can run the test with:
+
+```sh
+pip install pytest langfuse openai
+pytest test_geography_experiment.py -v
+```
+
+## Running Tests in CI/CD
+
+You can integrate these tests into your continuous integration pipeline.
+
+### GitHub Actions Example
+
+First, setup the environment variables in your Github Actions secrets:
+
+
+
+
+```yaml
+name: LLM Application Tests
+
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+ branches: [ main ]
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.10'
+
+ - name: Install dependencies
+ run: |
+ pip install pytest langfuse openai
+
+ - name: Run LLM unit tests
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }}
+ LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }}
+ LANGFUSE_HOST: ${{ secrets.LANGFUSE_HOST }}
+ run: |
+ pytest test_geography_experiment.py -v
+```
+
+## Using Remote Datasets with LLM-as-a-Judge
+
+For more advanced testing, you can use [remote datasets](https://langfuse.com/docs/evaluation/experiments/datasets) stored in Langfuse:
+
+```python {11,12}
+import pytest
+from langfuse import get_client, Langfuse
+
+@pytest.fixture
+def langfuse_client() -> Langfuse:
+ return get_client()
+
+def test_with_remote_dataset(langfuse_client: Langfuse):
+ """Test using a dataset stored in Langfuse with LLM-as-a-judge evaluation"""
+
+ # Fetch dataset from Langfuse
+ dataset = langfuse_client.get_dataset("geography-questions")
+
+ def task(*, item, **kwargs):
+ question = item.input
+ response = OpenAI().chat.completions.create(
+ model="gpt-4",
+ messages=[{"role": "user", "content": question}]
+ )
+ return response.choices[0].message.content
+
+ # Run experiment - Langfuse automatically applies configured evaluators
+ result = dataset.run_experiment(
+ name="Geography Test with Remote Dataset",
+ description="Testing geography QA with LLM-as-a-judge",
+ task=task
+ )
+
+ # The LLM-as-a-judge evaluator runs automatically in Langfuse
+ # Results are visible in the Langfuse UI
+ langfuse_client.flush()
+```
+
+You can then see the result of LLM-as-a-judge evaluation and the aggregated score in the Langfuse UI:
+
+
+
+### Benefits of Remote Datasets
+
+1. **Centralized test management**: Update test cases without code changes
+2. **LLM-as-a-judge evaluators**: Configure semantic evaluation in the Langfuse UI (see [LLM-as-a-judge](/docs/evaluation/evaluation-methods/llm-as-a-judge))
+3. **Historical tracking**: Compare results across runs to detect regressions
+4. **Team collaboration**: Share datasets across team members
+
+When using remote datasets, all experiment results are tracked in Langfuse:
+
+- View individual test case results with scores and reasoning
+- Compare experiment runs over time
+- Track which code changes improved or degraded performance
+- Share results with team members through the Langfuse dashboard
+
+We published two guides covering how to test chatbots and conversational applications with remote datasets and LLM-as-a-judge evaluators:
+
+- **[Evaluating Multi-Turn Conversations](https://langfuse.com/guides/cookbook/example_evaluating_multi_turn_conversations)**: Test specific points in ongoing conversations (N+1 evaluations), useful for debugging issues where context is lost across turns
+- **[Simulating Multi-Turn Conversations](https://langfuse.com/guides/cookbook/example_simulated_multi_turn_conversations)**: Generate synthetic conversations with AI agents to test various scenarios and personas
+
+## Next Steps
+
+Ready to implement automated testing for your LLM application? Start by [identifying 3-5 critical functionalities](/blog/2025-08-29-error-analysis-to-evaluate-llm-applications), create a small dataset with expected behaviors, and run your first experiment. We'd love to hear about your testing approach—join the conversation in our [GitHub Discussion](https://github.com/langfuse/langfuse/discussions), and explore our learning resources below.
+
+## Learn More
+
+import { FileCode, BookOpen, Video, Users, Joystick } from "lucide-react";
+
+
+ } arrow />
+ } arrow />
+ } arrow />
+ } arrow />
+
+
+## FAQ
+
+
+How do you test LLM applications?
+
+Testing LLM applications requires a different approach than traditional software testing. Instead of checking exact output matches, you use **evaluation functions** that score outputs on continuous scales. The typical approach combines three components: (1) **Datasets** — collections of input/output pairs representing test cases, (2) **Experiment runners** — tools that execute your LLM application against the dataset, and (3) **Evaluators** — scoring functions that assess output quality using LLM-as-a-Judge, semantic similarity, or custom logic. A test passes when evaluation scores meet your defined thresholds.
+
+
+
+
+What is the difference between LLM testing and LLM evaluation?
+
+**Testing** produces binary pass/fail results — your test suite tells you whether something broke. **Evaluation** measures quality on continuous scales — how accurate, helpful, or relevant are the responses. In practice, LLM testing and evaluation blend together: you "test" your application by "evaluating" its outputs with scoring functions, and a test passes if the evaluation score meets your threshold. Both are essential for building reliable LLM applications.
+
+
+
+
+How do I automate LLM testing in CI/CD?
+
+You can integrate LLM testing into your CI/CD pipeline using Langfuse's experiment runner SDK. Create a test script that: (1) loads your dataset (locally or from Langfuse), (2) runs your application against each test case, (3) scores the outputs with evaluator functions, and (4) asserts that scores meet minimum thresholds. Run this script as part of your CI pipeline using `pytest` or your preferred test runner. See the [experiments via SDK guide](/docs/evaluation/experiments/experiments-via-sdk) for implementation details.
+
+
diff --git a/content/academy/closing-the-loop.mdx b/content/academy/closing-the-loop.mdx
new file mode 100644
index 000000000..b81067142
--- /dev/null
+++ b/content/academy/closing-the-loop.mdx
@@ -0,0 +1,8 @@
+---
+title: Closing the Loop
+description: TODO
+---
+
+# Closing the Loop
+
+TODO: Closing the loop by looking at traces, selecting some for datasets, running evals on it and improving with prompt iteration
diff --git a/content/academy/cost-tracking.mdx b/content/academy/cost-tracking.mdx
new file mode 100644
index 000000000..6add75919
--- /dev/null
+++ b/content/academy/cost-tracking.mdx
@@ -0,0 +1,4 @@
+---
+title: Cost Tracking
+description: Langfuse tracks usage and cost of LLM generations for various models. Learn how to ingest, infer, and manage cost data.
+---
diff --git a/content/academy/designing-evals.mdx b/content/academy/designing-evals.mdx
new file mode 100644
index 000000000..cb8ab57d4
--- /dev/null
+++ b/content/academy/designing-evals.mdx
@@ -0,0 +1,110 @@
+---
+title: Designing Evals
+description: A practical guide to setting up automated evaluations for LLM applications and AI agents.
+---
+
+# Designing Good Evals
+
+In AI development, iterating quickly is important. Whether you're refining a prompt, swapping a model, or changing your application logic, you need to understand the impact of each change on performance. Manually annotating outputs after every modification is slow and expensive, especially when you want to integrate evaluations into a CI/CD pipeline.
+
+**Automated evaluators** solve this problem by providing a scalable way to measure and monitor your application's failure modes, enabling a fast and effective development loop.
+
+_The framework in this guide is adapted from Hamel Husain's [Eval FAQ](https://hamel.dev/blog/posts/evals-faq/)._
+
+---
+
+This guide describes a process to **build automated evaluators** for your application. This is a robust evaluator that you can scale for different tests and evolutions of your application:
+
+1. [What to Measure](#what-to-measure)
+2. [How to Measure](#how-to-measure)
+3. [Draft your LLM-as-a-Judge prompt](#draft-prompt)
+4. [Validate your evaluator](#validate-evaluator)
+
+---
+
+I'll demonstrate this process using an [example chatbot in the Langfuse documentation](/docs/demo) that uses the Vercel AI SDK and has access to a RAG tool to retrieve documents from the Langfuse documentation. The example chat app logs traces into the Langfuse example project and has already answered 19k user queries in the past year.
+
+Here's the chat interface (you can find the example chat app [here](/docs/demo)):
+
+
+ 
+
+
+## What to Measure [#what-to-measure]
+
+In the [previous blog post](/blog/2025-08-29-error-analysis-to-evaluate-llm-applications), we showed how to perform **error analysis** and identify failure modes in your application. Now we will focus on building automated evaluators to measure these failure modes.
+
+Before building an evaluator, it's important to differentiate between two types of failures to prioritize your efforts:
+
+**Missing Instructions:** The first type are errors caused by vague or incomplete instructions in your prompt. For instance if your agent uses too many bullet points or doesn't ask follow-up questions, and you never instructed it to do so, the first step is to fix the prompt. Creating an evaluator for a failure that a simple prompt tweak can solve is often unnecessary effort.
+
+**Model Limitations:** The second type occur when the LLM fails to perform correctly despite receiving clear and precise instructions. These are the ideal candidates for automated evaluation because they represent the model's inherent limitations, not a misunderstanding of your intent.
+
+Let's apply this to the [Langfuse Example App](/docs/demo). First, we fix some obvious issues by clarifying the prompt: "use a maximum of three bullet points" and "ask for more context when the user's query is ambiguous." With those fixed, we can focus on measuring the more complex model limitation failures we identified in the [previous blog post](/blog/2025-08-29-error-analysis-to-evaluate-llm-applications):
+
+* **Out of Scope**: The agent answers a question not related to Langfuse or the LLM/AI space.
+* **Generic Responses**: The answer is technically correct but doesn't resolve the user's issue. The metric is to assess if the agent's final answer is helpful and directly addresses the user's question.
+* **Context Retrieval / RAG Issues**: The agent uses the wrong retrieval tool. The metric needs to judge if the correct tool was chosen based on the user's query.
+
+For this guide, we will set up an evaluator for the "Out of Scope" failure mode.
+
+## How to Measure [#how-to-measure]
+
+In Langfuse, all evaluations are tracked as **Scores**, which [can be attached to traces, observations, sessions or dataset runs](/docs/evaluation/experiments/data-model#scores). Evaluations in Langfuse can be set up in two main ways:
+
+**In the Langfuse UI:** In Langfuse, you can set up **LLM-as-a-Judge Evaluators** that use another LLM to evaluate your application's output on subjective and nuanced criteria. These are easily configured directly in Langfuse. For a guide on setting them up in the UI, check the documentation on **[LLM-as-a-Judge evaluators](/docs/evaluation/evaluation-methods/llm-as-a-judge)**.
+
+**External Evaluators:** In your code, you can set up **Custom Evaluators** and use the Langfuse SDKs to send the scores back to the evaluated traces. This allows you to set up code-based evaluators or any other custom evaluation logic. For an example of a custom pipeline, see the guide on **[setting up an external evaluation pipelines](/guides/cookbook/example_external_evaluation_pipelines)**.
+
+In this guide, we will set up an LLM-as-a-Judge evaluator in the Langfuse UI.
+
+## Drafting your LLM-as-a-Judge Prompt [#draft-prompt]
+
+A good LLM-as-a-Judge prompt is narrowly focused and well-structured:
+
+1. **Pick one Failure Mode**: Focus on one specific failure mode. Do not try to cover multiple failure modes at once.
+2. **Pass/Fail Definitions**: Clearly define what constitutes a "Pass" (failure is absent) and a "Fail" (failure is present).
+3. **Examples**: Include clear examples of both "Pass" and "Fail" cases.
+
+Here is an example prompt I use in our Example App to check if the agent's answer is within its scope:
+
+
+ 
+
+
+Once you drafted your prompt, you can set up the LLM-as-a-Judge evaluator in the Langfuse UI. You can find a guide on how to set them up in the UI [here](/docs/evaluation/evaluation-methods/llm-as-a-judge).
+
+
+ 
+
+
+## Validating Your Evaluator [#validate-evaluator]
+
+To build an evaluator you can trust, you must validate its performance against human judgment, a process similar to testing a machine learning classifier.
+
+First, split a set of human-labeled traces into a **development set** and a **test set**. In Langfuse, you can use tags to manage these sets. The **development set** is used to iteratively refine your judge's prompt. Run the evaluator on this set, compare its scores to the human labels, analyze the disagreements, and adjust the prompt's definitions or examples until its judgments closely align with yours:
+
+
+ 
+
+
+Additionally, you can measure the judge's alignment with human labels. The best metrics for this are **True Positive Rate (TPR)** and **True Negative Rate (TNR)**. TPR measures what fraction of actual "Passes" your judge correctly identifies, while TNR measures what fraction of actual "Fails" it correctly identifies.
+
+You can calculate these metrics by querying the data from the trace table (via [UI export](/docs/api-and-data-platform/features/export-from-ui) or [SDKs](/docs/api-and-data-platform/features/query-via-sdk)) and calculating the metrics.
+
+Once your judge performs well on the dev set (e.g., TPR and TNR \>90%), run it a final time on the held-out **test set** to get an unbiased measure of its real-world performance. A validated evaluator with high TPR and TNR gives you confidence that your automated metrics are meaningful.
+
+You can now repeat this process for both your Evaluators in the Langfuse UI and your custom evaluators as part of an [external evaluation pipeline](/guides/cookbook/example_external_evaluation_pipelines).
+
+## Next Steps [#operationalize-evaluator]
+
+With good automated evaluators in place, the next step is to operationalize your workflow. The goal is to create a CI/CD pipeline where every code change (to a prompt, model, or tool) automatically triggers an evaluation run on a **golden [Langfuse Dataset](/docs/datasets)**.
+
+Your automated evaluators will score these runs, providing immediate feedback on how your changes impacted key failure modes. This continuous monitoring loop helps you develop faster while maintaining a high quality bar for your application.
+
+import { FileCode } from "lucide-react";
+
+
+ } arrow />
+
diff --git a/content/academy/evaluation-example-setups.mdx b/content/academy/evaluation-example-setups.mdx
new file mode 100644
index 000000000..286ca76c6
--- /dev/null
+++ b/content/academy/evaluation-example-setups.mdx
@@ -0,0 +1,12 @@
+---
+title: Example Setups
+description: Evaluation approaches for different types of LLM applications.
+---
+
+# Example Setups
+
+TODO
+
+Add example evaluator setups for
+- customer support chatbot (and link to demo project)
+- ...
diff --git a/content/academy/evaluation-loop.mdx b/content/academy/evaluation-loop.mdx
new file mode 100644
index 000000000..2ff1ec2df
--- /dev/null
+++ b/content/academy/evaluation-loop.mdx
@@ -0,0 +1,132 @@
+---
+title: The Evaluation Loop
+description: Learn the fundamental concepts behind LLM evaluation — the evaluation loop, evaluation methods, experiments, and online evaluation.
+---
+
+# The Evaluation Loop
+
+LLM applications often have a constant loop of testing and monitoring.
+
+**Offline evaluation** lets you test your application against a fixed dataset before you deploy. You run your new prompt or model against test cases, review the scores, iterate until the results look good, then deploy your changes. In Langfuse, you can do that by running [Experiments](/docs/evaluation/core-concepts#experiments).
+
+**Online evaluation** scores live traces to catch issues in real traffic. When you find edge cases your dataset didn't cover, you add them back to your dataset so future experiments will catch them.
+
+
+
+> **Here's an example workflow** for building a customer support chatbot
+> 1. You update your prompt to make responses less formal.
+> 2. Before deploying, you run an **experiment**: test the new prompt against your dataset of customer questions **(offline evaluation)**.
+> 3. You review the scores and outputs. The tone improved, but responses are longer and some miss important links.
+> 4. You refine the prompt and run the experiment again.
+> 5. The results look good now. You deploy the new prompt to production.
+> 6. You monitor with **online evaluation** to catch any new edge cases.
+> 7. You notice that a customer asked a question in French, but the bot responded in English.
+> 8. You add this French query to your dataset so future experiments will catch this issue.
+> 9. You update your prompt to support French responses and run another experiment.
+>
+> Over time, your dataset grows from a couple of examples to a diverse, representative set of real-world test cases.
+
+## Evaluation Methods [#evaluation-methods]
+
+Evaluation methods are the functions that score traces, observations, sessions, or dataset runs. You can use a variety of evaluation methods to add [scores](/docs/evaluation/experiments/data-model#scores).
+
+
+| Method | What | Use when |
+| --- | --- | --- |
+| [LLM-as-a-Judge](/docs/evaluation/evaluation-methods/llm-as-a-judge) | Use an LLM to evaluate outputs based on custom criteria | Subjective assessments at scale (tone, accuracy, helpfulness) |
+| [Scores via UI](/docs/evaluation/evaluation-methods/scores-via-ui) | Manually add scores to traces directly in the Langfuse UI | Quick quality spot checks, reviewing individual traces |
+| [Annotation Queues](/docs/evaluation/evaluation-methods/annotation-queues) | Structured human review workflows with customizable queues | Building ground truth, systematic labeling, team collaboration |
+| [Scores via API/SDK](/docs/evaluation/evaluation-methods/scores-via-sdk) | Programmatically add scores using the Langfuse API or SDK | Custom evaluation pipelines, deterministic checks, automated workflows |
+
+When setting up new evaluation methods, you can use [Score Analytics](/docs/evaluation/evaluation-methods/score-analytics) to analyze or sense-check the scores you produce.
+## Experiments [#experiments]
+
+An experiment runs your application against a dataset and evaluates the outputs. This is how you test changes before deploying to production.
+
+### Definitions
+
+Before diving into experiments, it's helpful to understand the building blocks in Langfuse: datasets, dataset items, tasks, scores, and experiments.
+
+| Object | Definition |
+| --- | --- |
+| **Dataset** | A collection of test cases (dataset items). You can run experiments on a dataset. |
+| **Dataset item** | One item in a dataset. Each dataset item contains an input (the scenario to test) and optionally an expected output. |
+| **Task** | The application code that you want to test in an experiment. This will be performed on each dataset item, and you will score the output.
+| **Evaluation Method** | A function that scores experiment results. In the context of a Langfuse experiment, this can be a [deterministic check](/docs/evaluation/evaluation-methods/custom-scores), or [LLM-as-a-Judge](/docs/evaluation/evaluation-methods/llm-as-a-judge). |
+| **Score** | The output of an evaluation. This can be numeric, categorical, or boolean. See [Scores](/docs/evaluation/experiments/data-model#scores) for more details.|
+| **Experiment Run** | A single execution of your task against all items in a dataset, producing outputs (and scores). |
+
+You can find the data model for these objects [here](/docs/evaluation/experiments/data-model).
+
+
+### How these work together
+
+This is what happens conceptually:
+
+When you run an experiment on a given **dataset**, each of the **dataset items** will be passed to the **task function** you defined. The task function is generally an LLM call that happens in your application, that you want to test. The task function produces an output for each dataset item. This process is called an **experiment run**. The resulting collection of outputs linked to the dataset items are the **experiment results**.
+
+Often, you want to score these experiment results. You can use various [evaluation methods](#evaluation-methods) that take in the dataset item and the output produced by the task function, and produce a score based on criteria you define. Based on these scores, you can then get a complete picture of how your application performs across all test cases.
+
+
+ 
+
+
+You can compare experiment runs to see if a new prompt version improves scores, or identify specific inputs where your application struggles. Based on these experiment results, you can decide whether the change is ready to be deployed to production.
+
+You can find more details on how these objects link together under the hood on the [data model page](/docs/evaluation/experiments/data-model).
+
+
+### Two ways to run experiments
+
+You can **run experiments programmatically using the Langfuse SDK**. This gives you full control over the task, evaluation logic, and more. [Learn more about running experiments via SDK](/docs/evaluation/experiments/experiments-via-sdk).
+
+Another way is to **run experiments directly from the Langfuse interface** by selecting a dataset and prompt version. This is useful for quick iterations on prompts without writing code. [Learn more about running experiments via UI](/docs/evaluation/experiments/experiments-via-ui).
+
+
+
+ {/* Header row */}
+
+
+ **Langfuse Execution**
+
+
+ **Local/CI Execution**
+
+
+ {/* Langfuse Data row */}
+
+ **Langfuse Dataset**
+
+
+ [Experiments via UI](/docs/evaluation/experiments/experiments-via-ui)
+
+
+ [Experiments via SDK](/docs/evaluation/experiments/experiments-via-sdk)
+
+
+ {/* Local Data row */}
+
+ **Local Dataset**
+
+
+ Not supported
+
+
+ [Experiments via SDK](/docs/evaluation/experiments/experiments-via-sdk)
+
+
+
+
+
+*While it's optional, we recommend managing the underlying [Datasets](/docs/evaluation/experiments/datasets) in Langfuse as it allows for [1] In-UI comparison tables of different experiments on the same data and [2] Iteratively improve dataset based on production/staging traces.*
+
+## Online Evaluation [#online-evaluation]
+
+For online evaluation, you can configure evaluation methods to automatically score production traces. This helps you catch issues immediately.
+
+Langfuse currently supports LLM-as-a-Judge and human annotation checks for online evaluation. [Deterministic checks are on the roadmap](https://github.com/orgs/langfuse/discussions/6087).
+
+
+### Monitoring with dashboards
+
+Langfuse offers dashboards to monitor your application performance in real-time. You can also monitor scores in dashboards. You can find more details on how to use dashboards [here](/docs/metrics/features/custom-dashboards).
diff --git a/content/academy/evaluation-overview.mdx b/content/academy/evaluation-overview.mdx
new file mode 100644
index 000000000..3b63a878c
--- /dev/null
+++ b/content/academy/evaluation-overview.mdx
@@ -0,0 +1,16 @@
+---
+title: Evaluating LLM Applications
+description: Learn how to systematically evaluate LLM applications — from understanding what to measure, to designing evals, to building the datasets that make it all work.
+---
+
+# Evaluating LLM Applications
+
+Once your traces are set up, and you decided what the behavior of your agent should be, you can start setting up evaluations, or short "Evals". There are many different forms of evaluations, each having their own pro's and cons. Most setups will benefit from a combination of different evaluation forms.
+
+This section will walk you through that:
+
+- [The evaluation loop](/academy/evaluation-loop)
+- [What should you evaluate?](/academy/what-to-evaluate)
+- [How to design an eval](/academy/designing-evals)
+- [Building good datasets](/academy/building-datasets)
+- [Example setups](/academy/evaluation-example-setups)
diff --git a/content/academy/good-traces.mdx b/content/academy/good-traces.mdx
new file mode 100644
index 000000000..bba8df8f8
--- /dev/null
+++ b/content/academy/good-traces.mdx
@@ -0,0 +1,143 @@
+---
+title: What does a good trace look like?
+description: A guide to structuring your Langfuse traces for effective debugging, evaluation, and cost tracking.
+---
+
+import { Fan, Wrench } from "lucide-react";
+
+# What does a good trace look like?
+
+You see traces appearing in Langfuse, but how do you know if you've done it well? Here are a couple of things you can look at and optimize.
+
+## What's the scope of one trace?
+
+Langfuse's [data model](/docs/observability/data-model) has three levels of grouping: observations (individual steps) are grouped into traces via a `trace_id`, and traces can be grouped into sessions via a `session_id`.
+
+A trace represents one self-contained unit of work in your application. Good examples of a typical trace:
+- One chatbot turn (user sends a message, your app retrieves context, calls the LLM, returns a response)
+- One agent run (the agent receives a task, reasons, calls tools, and produces a result)
+- One pipeline execution (a document comes in, gets chunked, embedded, and stored)
+
+If multiple of these happen in sequence, e.g. a multi-turn conversation, or several agent runs that feed into a final report, that's where [sessions](/docs/observability/features/sessions) come in. Each step is its own trace, and the session ties them together.
+
+A trace shows up in the Langfuse UI as a trace tree and an [agent graph](/docs/observability/features/agent-graphs):
+
+
+
+## Look at the trace tree
+
+When you click on a trace, you see the trace tree. There are two things you can check:
+
+### Are the right steps showing up?
+
+You should see your LLM calls, tool calls, and other important steps represented in the tree. They should have the correct [observation type](/docs/observability/features/observation-types).
+
+For example
+- an LLM call should show up as a `generation`. This is important because a `generation` can carry [cost, token usage, and model information](#track-model-tokens-and-cost-on-generations).
+- a tool call should show up as a `tool`. You can then filter on tool call observations when you create [LLM-as-a-judge](/docs/evaluation/evaluation-methods/llm-as-a-judge) evaluators.
+
+Framework integrations typically set these types automatically. If you're instrumenting manually, you can set them via the `as_type` parameter (Python) or `asType` (JS/TS). See the [observation types docs](/docs/observability/features/observation-types) for a full list.
+
+### Is there noise you don't need?
+
+Not every observation in the tree is useful for understanding what your application did. HTTP spans, database queries, and framework internals often add clutter without giving you meaningful insight. If you see observations like these polluting your trace tree, you can [filter them out](/faq/all/unwanted-http-database-spans).
+
+
+
+
+
+## Choose good names
+
+Observation- and trace names are used in many places:
+
+- When setting up [LLM-as-a-judge evaluators](/docs/evaluation/evaluation-methods/llm-as-a-judge), you target specific observations by name.
+- In [dashboards](/docs/metrics/features/custom-dashboards), you can filter and aggregate metrics by observation name.
+- In the tracing table, names help you quickly identify what each step does.
+
+Try to name the observations after what they do: `classify-intent`, `generate-response`, `summarize-results`. This makes it easier to understand what each step does when you're looking at the trace tree, and makes filtering on a specific step easier.
+
+> Try not to name observations after the AI model used (`gpt-4o`, `claude-sonnet`). It breaks as soon as you swap models. The model is already a separate attribute on `generation` observations, use that instead.
+
+## Choose meaningful input and output
+
+In general, it's recommended that operations have an input and/or output. If an observation has neither, ask yourself if an observation is actually useful or if you can drop it.
+
+For your **most viewed observations**, take some extra care to set them up. You will likely create pre-filtered views on your tracing and session screens. The observations you filter for here are the ones that will get looked at the most. For these, ask yourself: **what do I need to see to quickly evaluate a trace/session at a glance?**
+
+
+
+Typical input/output for `GENERATION` observations:
+- For a chatbot: the user message (input) and the assistant response (output).
+- For a RAG pipeline: the user query and the generated answer.
+- For a classification task: the text being classified and the predicted label.
+
+If your input and output fields are showing up empty unintentionally, see [why are the input and output of my trace empty?](/faq/all/empty-trace-input-and-output)
+
+## Useful attributes
+
+Observations have a number of attributes that can be useful for your use cases. These will allow you to go even further with filtering, scoring, and making dashboards.
+
+### Add metadata for context
+
+[Metadata](/docs/observability/features/metadata) is a flexible key-value store on each observation. Some data that might be useful to save under metadata:
+
+- **Evaluation context**: When configuring [LLM-as-a-judge evaluators](/docs/evaluation/evaluation-methods/llm-as-a-judge), you can reference metadata fields. This is useful for passing ground truth, expected behavior, or other context that the evaluator needs but that isn't part of the actual input/output.
+- **Filtering**: You can filter by metadata keys in the Langfuse UI, which is helpful when you need to find traces with specific characteristics.
+- **Annotation context**: When doing manual review, metadata gives annotators extra information to make better judgments.
+
+### Track model, tokens, and cost on generations [#track-model-tokens-and-cost-on-generations]
+
+If you want to understand what your LLM usage costs, broken down by model, by user, by feature, you need three things on your `generation` observations:
+
+- **Model name**: Langfuse uses this to look up pricing in the [model pricing table](/docs/model-usage-and-cost). If the model name doesn't match, Langfuse can't calculate cost automatically.
+- **Usage details**: Input tokens, output tokens, and optionally cached tokens. This is what powers the token usage views in dashboards.
+- **Cost details** (optional): If you want to override Langfuse's automatic pricing — for example, if you have a custom pricing agreement — you can pass cost explicitly.
+
+Most [integrations](/integrations) capture all of this automatically. If you're instrumenting manually, see the [token and cost tracking docs](/docs/observability/features/token-and-cost-tracking).
+
+You can see these attributes on the `GENERATION` observation in the Langfuse UI.
+
+
+
+### Use tags for business-level dimensions
+
+[Tags](/docs/observability/features/tags) enable filtering and metric breakdowns across dimensions that matter to your business. Good tags answer questions like "how does latency differ between our `web` and `api` users?".
+
+One property of tags is that they are **immutable and must be set at observation creation time**. This makes them great for things you know upfront (where the request came from, which feature it's part of), but not for things you learn later.
+
+If you need to label traces based on something determined after the fact, like an [LLM-as-a-judge](/docs/evaluation/evaluation-methods/llm-as-a-judge) evaluation result, use [scores](/docs/evaluation/overview) instead.
+
+### Link prompts to traces
+
+If you [manage your prompts in Langfuse](/docs/prompt-management), you can [link them to your generations](/docs/prompt-management/features/link-to-traces). This lets you see which prompt version was used for a given trace, and track how metrics change across prompt versions. Useful when you're iterating on prompts and want to compare performance.
+
+### Set the environment
+
+Set the [environment](/docs/observability/features/environments) attribute (`production`, `staging`, `development`) so that your test traces don't pollute production dashboards and evaluations.
+
+### Track users with user IDs
+
+Setting the [user ID](/docs/observability/features/users) connects traces to specific users, which unlocks per-user views in Langfuse. Useful if you want to answer questions like:
+
+- Which users are costing us the most?
+- How does output quality vary across users?
+- What does a specific user's usage pattern look like?
+
+### Group related traces with session IDs
+
+If your application involves multiple traces that logically belong together, group them into a [session](/docs/observability/features/sessions). This gives you a session replay view where you can see the full interaction in sequence.
+
+This makes sense when:
+- You're building a chatbot (each user message creates a new trace, but the whole conversation is one session)
+- You have multiple agents that each contribute to a final output (e.g., five agents that collaborate to produce a report)
+- Your workflow spans multiple requests with human-in-the-loop steps in between
+
+If your application is single-request/single-response with no continuity between calls, you probably don't need sessions.
+
+
diff --git a/content/academy/index.mdx b/content/academy/index.mdx
new file mode 100644
index 000000000..2bad58e2f
--- /dev/null
+++ b/content/academy/index.mdx
@@ -0,0 +1,39 @@
+---
+title: Langfuse Academy
+description: Learn how to build, evaluate, and improve LLM applications. Conceptual guides on observability, evaluation, prompt management, and production monitoring.
+---
+
+# Langfuse Academy
+
+The Academy is a collection of conceptual guides that help you understand how to build a good setup for your LLM application. It's not about how to integrate a specific SDK — that's what the [docs](/docs) are for. Instead, it covers the thinking behind the decisions: what does good look like, what processes should you follow, and what should you keep in mind?
+
+## Sections
+
+import { Eye, FlaskConical, MessageSquareText, Activity } from "lucide-react";
+
+
+ }
+ title="LLM Observability"
+ description="How LLM observability differs from traditional observability, core concepts, and what good traces look like."
+ href="/academy/observability-overview"
+ />
+ }
+ title="Evaluating LLM Applications"
+ description="The evaluation loop, choosing what to evaluate, designing evals, building datasets, and example setups."
+ href="/academy/evaluation-overview"
+ />
+ }
+ title="Prompt Management"
+ description="When prompt management makes sense, version control, deployment strategies, and composability."
+ href="/academy/prompt-management-overview"
+ />
+ }
+ title="Monitoring & Improving in Production"
+ description="What to monitor, cost tracking, user feedback, closing the loop, and security guardrails."
+ href="/academy/monitoring-overview"
+ />
+
diff --git a/content/academy/llm-observability-vs-traditional.mdx b/content/academy/llm-observability-vs-traditional.mdx
new file mode 100644
index 000000000..e76e2d8db
--- /dev/null
+++ b/content/academy/llm-observability-vs-traditional.mdx
@@ -0,0 +1,8 @@
+---
+title: How Is LLM Observability Different?
+description: TODO
+---
+
+# How Is LLM Observability Different?
+
+TODO
diff --git a/content/academy/meta.json b/content/academy/meta.json
new file mode 100644
index 000000000..8d22e9d41
--- /dev/null
+++ b/content/academy/meta.json
@@ -0,0 +1,32 @@
+{
+ "title": "Academy",
+ "pages": [
+ "index",
+ "---LLM Observability---",
+ "observability-overview",
+ "llm-observability-vs-traditional",
+ "observability-core-concepts",
+ "good-traces",
+ "observability-use-case-examples",
+ "---Evaluating LLM Applications---",
+ "evaluation-overview",
+ "evaluation-loop",
+ "what-to-evaluate",
+ "designing-evals",
+ "building-datasets",
+ "evaluation-example-setups",
+ "---Prompt Management---",
+ "prompt-management-overview",
+ "prompt-management-when-it-makes-sense",
+ "prompt-version-control",
+ "prompt-deployment-strategies",
+ "prompt-composability",
+ "---Monitoring & Improving in Production---",
+ "monitoring-overview",
+ "what-to-monitor",
+ "cost-tracking",
+ "user-feedback",
+ "closing-the-loop",
+ "security-and-guardrails"
+ ]
+}
diff --git a/content/academy/monitoring-overview.mdx b/content/academy/monitoring-overview.mdx
new file mode 100644
index 000000000..bc94067ec
--- /dev/null
+++ b/content/academy/monitoring-overview.mdx
@@ -0,0 +1,14 @@
+---
+title: Monitoring & Improving in Production
+description: You shipped your LLM application — now what? Learn what to monitor, how to track costs, collect feedback, and continuously improve.
+---
+
+# Monitoring & Improving in Production
+
+You shipped your LLM application — now learn what to monitor, how to track costs, collect feedback, and continuously improve.
+
+- [What to monitor](/academy/what-to-monitor)
+- [Cost tracking and optimization](/academy/cost-tracking)
+- [Collecting and using user feedback](/academy/user-feedback)
+- [Closing the loop](/academy/closing-the-loop)
+- [Security and guardrails](/academy/security-and-guardrails)
diff --git a/content/academy/observability-core-concepts.mdx b/content/academy/observability-core-concepts.mdx
new file mode 100644
index 000000000..024a1588d
--- /dev/null
+++ b/content/academy/observability-core-concepts.mdx
@@ -0,0 +1,151 @@
+---
+title: Observability Core Concepts
+description: Understand how Langfuse structures traces, observations, and sessions — the building blocks of LLM observability.
+---
+
+# Core Concepts
+
+This page digs into the underlying concepts of how Langfuse structures and captures your data. Understanding these will make setting up and working with your traces easier.
+
+## Observations, Traces, and Sessions
+
+Langfuse organizes an application's data into three core concepts: observations, traces, and sessions.
+
+```mermaid
+flowchart LR
+
+ subgraph SESSION [session]
+ direction TB
+
+ subgraph TRACE1 [trace]
+ direction LR
+ O1[observation]
+ O2[observation]
+ O3[observation]
+ end
+
+ subgraph TRACE2 [trace]
+ direction LR
+ O4[observation]
+ O5[observation]
+ O6[observation]
+ end
+
+ subgraph TRACE3 [trace]
+ direction LR
+ O7[observation]
+ O8[observation]
+ O9[observation]
+ end
+
+ end
+ classDef sessionBox fill:none,stroke:#888,stroke-width:2px;
+ classDef traceBox fill:none,stroke:#aaa,stroke-width:1.5px;
+
+ %% Assign classes
+ class SESSION sessionBox;
+ class TRACE1,TRACE2,TRACE3 traceBox;
+```
+
+import ObservationTypesList from "@/components-mdx/observation-types-list.mdx";
+
+### Observations
+
+`Observations` are the individual steps within a trace. Langfuse supports a number of LLM application specific [observation types](/docs/observability/features/observation-types), for example _generations_, _toolcalls_, _RAG retrieval steps_, etc.
+
+Observations can be nested. The example below shows a trace with a nested observation.
+
+
+
+
+
+Hierarchical structure of observations in Langfuse
+
+
+
+Example trace in Langfuse UI
+
+
+
+
+
+
+
+Example trace in Langfuse UI
+
+
+
+
+
+
+
+### Traces
+
+A `trace` typically represents a single request or operation.
+For example, when a user asks a question to a chatbot, that interaction, from the user's question to the bot's response, is captured as one trace.
+
+It serves as container of observations. Trace attributes such as `user_id`, `session_id`, `tags`, `metadata`, etc. are propagated to all observations within the trace.
+
+### Sessions
+
+Optionally, traces can be grouped into [sessions](/docs/observability/features/sessions).
+Sessions are used to group traces that are part of the same user interaction.
+A common example is a thread in a chat interface.
+
+
+
+Using sessions is recommended for applications with multi-turn conversations or workflows. Please refer to the [Sessions](/docs/observability/features/sessions) documentation to add sessions to your traces.
+
+## Adding Attributes
+
+Once you've structured your data into traces and observations, you can enrich them with additional attributes. These attributes act as labels that help you filter, segment, and analyze your traces for specific use cases.
+
+There are different types of attributes you can add:
+
+| Attribute | Description |
+|-----------|-------------|
+| [Environments](/docs/observability/features/environments) | Separate data from different deployment contexts like `production`, `staging`, or `development` |
+| [Tags](/docs/observability/features/tags) | Flexible labels to categorize traces by feature, API endpoint, or workflow |
+| [User](/docs/observability/features/users) | Track which end-user triggered each trace |
+| [Metadata](/docs/observability/features/metadata) | Flexible key-value store for custom information |
+| [Releases & Versions](/docs/observability/features/releases-and-versioning) | Track application versions and component changes |
+
+The next page goes deeper into when it makes sense to add certain attributes.
+
diff --git a/content/academy/observability-overview.mdx b/content/academy/observability-overview.mdx
new file mode 100644
index 000000000..a7f9860fb
--- /dev/null
+++ b/content/academy/observability-overview.mdx
@@ -0,0 +1,15 @@
+---
+title: LLM Observability
+description: Understand what LLM observability is, how it differs from traditional observability, and why it matters for building reliable AI applications.
+---
+
+# LLM Observability
+
+This section dives into LLM observability. It's the first step towards a solid AI setup and one you shouldn't wait on. Even if you don't feel ready for other aspects of LLM engineering, having traces of your agents is always a good idea. You can already study their behavior, make small fixes based on what you see, and form a more grounded opinion on what good looks like for you agent in specific situations.
+
+This section covers the following topics:
+
+- [How is LLM observability different from traditional logging?](/academy/llm-observability-vs-traditional)
+- [Core observability concepts in Langfuse](/academy/observability-core-concepts)
+- [What does a good trace look like?](/academy/good-traces)
+- [Use-case specific example setups](/academy/observability-use-case-examples)
diff --git a/content/academy/observability-use-case-examples.mdx b/content/academy/observability-use-case-examples.mdx
new file mode 100644
index 000000000..cbf5d2ca6
--- /dev/null
+++ b/content/academy/observability-use-case-examples.mdx
@@ -0,0 +1,12 @@
+---
+title: Use Case Examples
+description: TODO
+---
+
+# Use Case Specific Examples
+
+TODO:
+- refer to demo project
+- discuss
+ - a complicated customer support chatbot trace
+ - ...
diff --git a/content/academy/prompt-composability.mdx b/content/academy/prompt-composability.mdx
new file mode 100644
index 000000000..0b4ca1600
--- /dev/null
+++ b/content/academy/prompt-composability.mdx
@@ -0,0 +1,67 @@
+---
+title: Prompt Composability
+description: Reference other prompts in your prompts using a simple tag format to create modular, reusable prompt components.
+---
+
+import { FaqPreview } from "@/components/faq/FaqPreview";
+
+# Prompt Composability
+
+As you create more prompts, you will often find yourself using the same snippets of text or instructions in multiple prompts. To avoid duplication, you can compose prompts by referencing other prompts.
+
+
+
+## Why Use Composed Prompts?
+
+- Create modular **prompt components** that can be reused across multiple prompts
+- **Maintain** common instructions, examples, or context in a single place
+- **Update dependent prompts** automatically when base prompts change
+
+## Get started
+
+
+
+
+When creating the prompt via the Langfuse UI, you can use the `Add prompt reference` button to insert a prompt reference into your prompt.
+
+
+
+
+
+
+
+You can reference other **text** prompts in your prompts the following format:
+
+```bash
+@@@langfusePrompt:name=PromptName|version=1@@@
+```
+
+You can also use a label instead of a specific version for dynamic resolution:
+
+```bash
+@@@langfusePrompt:name=PromptName|label=production@@@
+```
+
+
+
+
+
+
+
+Not exactly what you need? Consider these similar features:
+- [Variables](/docs/prompt-management/features/variables) for inserting dynamic text into prompts
+- [Message placeholders](/docs/prompt-management/features/message-placeholders) for inserting arrays of complete messages instead of strings
+
+Or related FAQ pages:
+
+
diff --git a/content/academy/prompt-deployment-strategies.mdx b/content/academy/prompt-deployment-strategies.mdx
new file mode 100644
index 000000000..98c322e3e
--- /dev/null
+++ b/content/academy/prompt-deployment-strategies.mdx
@@ -0,0 +1,8 @@
+---
+title: Deployment Strategies
+description: TODO
+---
+
+# Deployment Strategies
+
+{/* TODO */}
diff --git a/content/academy/prompt-management-overview.mdx b/content/academy/prompt-management-overview.mdx
new file mode 100644
index 000000000..a0db227ab
--- /dev/null
+++ b/content/academy/prompt-management-overview.mdx
@@ -0,0 +1,13 @@
+---
+title: Prompt Management as a Practice
+description: Learn when and how to manage prompts as a practice — version control, deployment strategies, and composability.
+---
+
+# Prompt Management as a Practice
+
+Treat prompts as first-class artifacts: versioned, deployable independently from code, and accessible to the people who need to iterate on them.
+
+- [When does prompt management make sense?](/academy/prompt-management-when-it-makes-sense)
+- [Why prompts deserve version control](/academy/prompt-version-control)
+- [Deployment strategies](/academy/prompt-deployment-strategies)
+- [Composability](/academy/prompt-composability)
diff --git a/content/academy/prompt-management-when-it-makes-sense.mdx b/content/academy/prompt-management-when-it-makes-sense.mdx
new file mode 100644
index 000000000..e5a834d37
--- /dev/null
+++ b/content/academy/prompt-management-when-it-makes-sense.mdx
@@ -0,0 +1,8 @@
+---
+title: When Does Prompt Management Make Sense?
+description: TODO
+---
+
+# When Does Prompt Management Make Sense?
+
+{/* TODO */}
diff --git a/content/academy/prompt-version-control.mdx b/content/academy/prompt-version-control.mdx
new file mode 100644
index 000000000..2e816bcea
--- /dev/null
+++ b/content/academy/prompt-version-control.mdx
@@ -0,0 +1,187 @@
+---
+title: Prompt Version Control
+description: Use prompt labels and versions to manage prompt deployments, rollbacks, and diffs in Langfuse.
+---
+
+# Prompt Version Control
+
+In Langfuse, version control & deployment of prompts is managed via `versions` and `labels`.
+
+## Versions & Labels
+
+Each prompt version is automatically assigned a `version ID`. Additionally, you can assign `labels` to follow your own versioning scheme.
+
+Labels can be used to assign prompts to environments (staging, production), tenants (tenant-1, tenant-2), or experiments (prod-a, prod-b).
+
+
+
+
+Use the Langfuse UI to assign labels to a prompt.
+
+
+
+
+
+
+Use the Python SDK to assign labels to a prompt when creating a new prompt version.
+
+```python {5}
+langfuse.create_prompt(
+ name="movie-critic",
+ type="text",
+ prompt="As a {{criticlevel}} movie critic, do you like {{movie}}?",
+ labels=["production"], # add the label "production" to the prompt version
+)
+```
+
+Alternatively, you can also update the labels of an existing prompt version using the Python SDK:
+
+```python {5}
+langfuse = Langfuse()
+langfuse.update_prompt(
+ name="movie-critic",
+ version=1,
+ new_labels=["john", "doe"], # assign these labels to the prompt version
+)
+```
+
+
+
+
+Use the JS/TS SDK to assign labels to a prompt when creating a new prompt version.
+
+```ts {5}
+import { LangfuseClient } from "@langfuse/client";
+
+const langfuse = new LangfuseClient();
+
+await langfuse.prompt.create({
+ name: "movie-critic",
+ type: "text",
+ prompt: "As a {{criticlevel}} critic, do you like {{movie}}?",
+ labels: ["production"], // add the label "production" to the prompt version
+});
+```
+
+Alternatively, you can also update the labels of an existing prompt version using the JS/TS SDK:
+
+```ts {5}
+await langfuse.prompt.update({
+ name: "movie-critic",
+ version: 1,
+ newLabels: ["john", "doe"],
+});
+```
+
+
+
+
+## Fetching by Label or Version
+
+When fetching prompts to use them in your application you can either do so by fetching a specific version or label.
+Here are code examples for fetching prompts by label or version.
+
+**To "deploy" a prompt version**, you have to assign the label `production` or any environment label you created to that prompt version.
+
+Some notes on fetching prompts:
+
+- The `latest` label points to the most recently created version.
+- When using a prompt without specifying a label, Langfuse will serve the version with the `production` label.
+
+
+
+
+```python
+from langfuse import get_client
+
+# Initialize Langfuse client
+langfuse = get_client()
+
+# Get specific version
+prompt = langfuse.get_prompt("movie-critic", version=1)
+
+# Get specific label
+prompt = langfuse.get_prompt("movie-critic", label="staging")
+
+# Get latest prompt version. The 'latest' label is automatically maintained by Langfuse.
+prompt = langfuse.get_prompt("movie-critic", label="latest")
+```
+
+
+
+
+
+```ts
+import { LangfuseClient } from "@langfuse/client";
+
+const langfuse = new LangfuseClient();
+
+// Get specific version of a prompt (here version 1)
+const prompt = await langfuse.prompt.get("movie-critic", {
+ version: 1,
+});
+
+// Get specific label
+const prompt = await langfuse.prompt.get("movie-critic", {
+ label: "staging",
+});
+
+// Get latest prompt version. The 'latest' label is automatically maintained by Langfuse.
+const prompt = await langfuse.prompt.get("movie-critic", {
+ label: "latest",
+});
+```
+
+
+
+
+
+## Rollbacks
+
+When a prompt has a `production` label, then that version will be served by default in the SDKs. You can quickly rollback to a previous version by setting the `production` label to that previous version in the Langfuse UI.
+
+## Prompt Diffs
+
+The prompt version diff view shows you the changes you made to the prompt over time. This helps you understand how the prompt has evolved and what changes have been made to debug issues or understand the impact of changes.
+
+
+
+## Protected prompt labels
+
+
+
+Protected prompt labels give project admins and owners ([RBAC docs](/docs/rbac)) the ability to prevent labels from being modified or deleted, ensuring better control over prompt deployment.
+
+Once a label such as `production` is marked as protected:
+
+- `viewer` and `member` roles cannot modify or delete the label from prompts, preventing changes to the `production` prompt version. This also blocks the deletion of the prompt.
+- `admin` and `owner` roles can still modify or delete the label, effectively changing the `production` prompt version.
+
+Admins and owners can update a label's protection status in the project settings.
+
+
+
+## Related Resources
+
+- Prompts are scoped to a project — if you use separate projects for different environments, see [how to sync prompts between them](/faq/all/managing-different-environments)
+- To compare prompt versions on a dataset before promoting a label, run [Experiments](/docs/evaluation/core-concepts#experiments).
diff --git a/content/academy/security-and-guardrails.mdx b/content/academy/security-and-guardrails.mdx
new file mode 100644
index 000000000..80ad3da3b
--- /dev/null
+++ b/content/academy/security-and-guardrails.mdx
@@ -0,0 +1,219 @@
+---
+title: Security & Guardrails
+description: "Comprehensive guide to LLM security guardrails — protect against prompt injection, PII leakage, and harmful content using LLM Guard, NeMo Guardrails, and Langfuse."
+---
+
+# LLM Security & Guardrails
+
+There are a host of potential safety risks involved with LLM-based applications. These include prompt injection, leakage of personally identifiable information (PII), or harmful prompts. Langfuse can be used to monitor and protect against these security risks, and investigate incidents when they occur.
+
+
+
+## What is LLM Security?
+
+LLM Security involves implementing protective measures to safeguard LLMs and their infrastructure from unauthorized access, misuse, and adversarial attacks, ensuring the integrity and confidentiality of both the model and data. This is crucial in AI/ML systems to maintain ethical usage, prevent security risks like prompt injections, and ensure reliable operation under safe conditions.
+
+## How does LLM Security work?
+
+
+
+LLM Security can be addressed with a combination of
+
+- LLM Security libraries for run-time security measures
+- Langfuse for the ex-post evaluation of the effectiveness of these measures
+
+
+
+### 1. Run-time security measures
+
+There are several popular security libraries that can be used to mitigate security risks in LLM-based applications. These include: [LLM Guard](https://llm-guard.com), [Prompt Armor](https://promptarmor.com), [NeMo Guardrails](https://github.com/NVIDIA/NeMo-Guardrails), [Microsoft Azure AI Content Safety](https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety), [Lakera](https://www.lakera.ai). These libraries help with security measures in the following ways:
+
+1. Catching and blocking a potentially harmful or inappropriate prompt before sending to the model
+2. Redacting sensitive PII before sending into the model and then un-redacting in the response
+3. Evaluating prompts and completions on toxicity, relevance, or sensitive material at run-time and blocking the response if necessary
+
+### 2. Monitoring and evaluation of security measures with Langfuse
+
+Use Langfuse [tracing](/docs/observability/overview) to gain visibility and confidence in each step of the security mechanism. These are common workflows:
+
+1. Manually inspect traces to investigate security issues.
+2. Monitor security scores over time in the Langfuse Dashboard.
+3. Validate security checks. You can use Langfuse [scores](/docs/evaluation/core-concepts) to evaluate the effectiveness of security tools. Integrating Langfuse into your team's workflow can help teams identify which security risks are most prevalent and build more robust tools around those specific issues. There are two main workflows to consider:
+ - [Annotations (in UI)](/docs/evaluation/evaluation-methods/annotation-queues). If you establish a baseline by annotating a share of production traces, you can compare the security scores returned by the security tools with these annotations.
+ - [Automated evaluations](/docs/evaluation/evaluation-methods/llm-as-a-judge). Langfuse's model-based evaluations will run asynchronously and can scan traces for things such as toxicity or sensitivity to flag potential risks and identify any gaps in your LLM security setup. Check out the docs to learn more about how to set up these evaluations.
+4. Track Latency. Some LLM security checks need to be awaited before the model can be called, others block the response to the user. Thus they quickly are an essential driver of overall latency of an LLM application. Langfuse can help dissect the latencies of these checks within a trace to understand whether the checks are worth the wait.
+
+## Getting Started
+
+> Example: Anonymizing Personally Identifiable Information (PII)
+
+Exposing PII to LLMs can pose serious security and privacy risks, such as violating contractual obligations or regulatory compliance requirements, or mitigating the risks of data leakage or a data breach.
+
+Personally Identifiable Information (PII) includes:
+
+- Credit card number
+- Full name
+- Phone number
+- Email address
+- Social Security number
+- IP Address
+
+The example below shows a simple application that summarizes a given court transcript. For privacy reasons, the application wants to anonymize PII before the information is fed into the model, and then un-redact the response to produce a coherent summary.
+
+To read more about other security risks, including prompt injection, banned topics, or malicious URLs, please check out the docs of the various libraries or read our [security cookbook](/docs/security/example-python) which includes more examples.
+
+
+
+### Install packages
+
+In this example we use the open source library [LLM Guard](https://protectai.github.io/llm-guard/) for run-time security checks. All examples easily translate to other libraries such as [Prompt Armor](https://promptarmor.com), [NeMo Guardrails](https://github.com/NVIDIA/NeMo-Guardrails), [Microsoft Azure AI Content Safety](https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety), and [Lakera](https://www.lakera.ai).
+
+First, import the security packages and Langfuse tools.
+
+```bash
+pip install llm-guard langfuse openai
+```
+
+```python
+from llm_guard.input_scanners import Anonymize
+from llm_guard.input_scanners.anonymize_helpers import BERT_LARGE_NER_CONF
+from langfuse.openai import openai # OpenAI integration
+from langfuse import observe
+from llm_guard.output_scanners import Deanonymize
+from llm_guard.vault import Vault
+```
+
+### Anonymize and deanonymize PII and trace with Langfuse
+
+We break up each step of the process into its own function so we can track each step separately in Langfuse.
+
+By decorating the functions with `@observe()`, we can trace each step of the process and monitor the risk scores returned by the security tools. This allows us to see how well the security tools are working and whether they are catching the PII as expected.
+
+```python
+vault = Vault()
+
+@observe()
+def anonymize(input: str):
+ scanner = Anonymize(vault, preamble="Insert before prompt", allowed_names=["John Doe"], hidden_names=["Test LLC"],
+ recognizer_conf=BERT_LARGE_NER_CONF, language="en")
+ sanitized_prompt, is_valid, risk_score = scanner.scan(prompt)
+ return sanitized_prompt
+
+@observe()
+def deanonymize(sanitized_prompt: str, answer: str):
+ scanner = Deanonymize(vault)
+ sanitized_model_output, is_valid, risk_score = scanner.scan(sanitized_prompt, answer)
+
+ return sanitized_model_output
+```
+
+### Instrument LLM call
+
+In this example, we use the native OpenAI SDK integration, to instrument the LLM call. Thereby, we can automatically collect token counts, model parameters, and the exact prompt that was sent to the model.
+
+Note: Langfuse [natively integrates](/integrations) with a number of frameworks (e.g. LlamaIndex, LangChain, Haystack, ...) and you can easily instrument any LLM via the [SDKs](/docs/sdk).
+
+```python
+@observe()
+def summarize_transcript(prompt: str):
+ sanitized_prompt = anonymize(prompt)
+
+ answer = openai.chat.completions.create(
+ model="gpt-4o-mini",
+ max_tokens=100,
+ messages=[
+ {"role": "system", "content": "Summarize the given court transcript."},
+ {"role": "user", "content": sanitized_prompt}
+ ],
+ ).choices[0].message.content
+
+ sanitized_model_output = deanonymize(sanitized_prompt, answer)
+
+ return sanitized_model_output
+```
+
+### Execute the application
+
+Run the function. In this example, we input a section of a court transcript. Applications that handle sensitive information will often need to use anonymize and deanonymize functionality to comply with data privacy policies such as HIPAA or GDPR.
+
+```python
+prompt = """
+Plaintiff, Jane Doe, by and through her attorneys, files this complaint
+against Defendant, Big Corporation, and alleges upon information and belief,
+except for those allegations pertaining to personal knowledge, that on or about
+July 15, 2023, at the Defendant's manufacturing facility located at 123 Industrial Way, Springfield, Illinois, Defendant negligently failed to maintain safe working conditions,
+leading to Plaintiff suffering severe and permanent injuries. As a direct and proximate
+result of Defendant's negligence, Plaintiff has endured significant physical pain, emotional distress, and financial hardship due to medical expenses and loss of income. Plaintiff seeks compensatory damages, punitive damages, and any other relief the Court deems just and proper.
+"""
+summarize_transcript(prompt)
+```
+
+### Inspect trace in Langfuse
+
+In this trace ([public link](https://cloud.langfuse.com/project/cloramnkj0002jz088vzn1ja4/traces/43213866-3038-4706-ae3a-d39e9df459a2)), we can see how the name of the plaintiff is anonymized before being sent to the model, and then un-redacted in the response. We can now evaluate run evaluations in Langfuse to control for the effectiveness of these measures.
+
+
+
+
+
+## More Examples
+
+Find more examples of LLM security monitoring in our cookbook.
+
+import { FileCode, BookOpen } from "lucide-react";
+
+
+ }
+ />
+
+
+## FAQ
+
+
+What is LLM Guard?
+
+[LLM Guard](https://llm-guard.com) is an open-source library by Protect AI that provides security guardrails for LLM applications. It includes input scanners (prompt injection detection, PII anonymization, toxicity detection, topic banning) and output scanners (content moderation, bias detection, malicious URL detection, deanonymization). LLM Guard can be integrated with Langfuse to trace and monitor the effectiveness of each security check.
+
+
+
+
+What are security guardrails for LLMs?
+
+Security guardrails are protective measures that intercept and filter inputs and outputs of LLM applications to prevent misuse and harm. They include input guardrails (blocking prompt injection and malicious inputs), output guardrails (filtering toxic or harmful responses), PII detection and redaction, content filtering, and topic restriction. Guardrails run at the application layer and are complementary to model-level safety training.
+
+
+
+
+How do I prevent prompt injection in my LLM application?
+
+Prompt injection can be mitigated through multiple layers: (1) Use a guardrail library like LLM Guard or Lakera to detect and block injection attempts before they reach the model. (2) Implement clear separation between system instructions and user input. (3) Use Langfuse tracing to monitor for injection attempts in production. (4) Set up automated evaluations to flag suspicious patterns. No single approach is 100% effective, so a defense-in-depth strategy with monitoring is recommended.
+
+
+
+
+How do I monitor LLM security in production?
+
+Use Langfuse to monitor your LLM security posture: (1) Trace every request through your guardrail pipeline to see which checks triggered. (2) Use LLM-as-a-Judge evaluators to automatically score traces for toxicity, PII leakage, and prompt injection. (3) Set up custom dashboards to track security scores over time. (4) Use annotation queues for human review of flagged conversations. (5) Track the latency impact of security checks to balance protection with user experience.
+
+
+
+## GitHub Discussions
+
+import { GhDiscussionsPreview } from "@/components/gh-discussions/GhDiscussionsPreview";
+
+
diff --git a/content/academy/user-feedback.mdx b/content/academy/user-feedback.mdx
new file mode 100644
index 000000000..833ad70d4
--- /dev/null
+++ b/content/academy/user-feedback.mdx
@@ -0,0 +1,135 @@
+---
+title: User Feedback
+description: Collect user feedback on LLM or agent outputs to improve model performance and user satisfaction.
+---
+
+# User Feedback
+
+User feedback measures whether your AI actually helped users. Use it to find quality issues, build better evaluation datasets, and prioritize improvements based on real user experiences. In Langfuse, feedback is captured as [scores](/docs/scores) and linked to traces.
+
+
+
+## Feedback Types
+
+### Explicit Feedback
+
+Users directly rate responses through thumbs up/down, star ratings, or comments.
+| Pros | Cons |
+|------|------|
+| Clear signal about satisfaction | Low response rates |
+| Simple to implement | Unhappy users more likely to respond |
+| Easy to act on | Requires user action |
+
+### Implicit Feedback
+
+Derived from user behavior like time spent reading, copying output, accepting suggestions, or retrying queries.
+
+| Pros | Cons |
+|------|------|
+| High volume on every interaction | Harder to implement |
+| No user effort required | Ambiguous signals |
+| Reflects actual usage | Requires interpretation |
+
+Both work as [scores](/docs/evaluation/core-concepts) in Langfuse. Filter traces by score, build [annotation queues](/docs/evaluation/evaluation-methods/annotation-queues), or use feedback as ground truth for automated evaluations.
+
+## Quick Start
+
+This example shows how to collect explicit user feedback from a chatbot built with Next.js and AI SDK. You can find the full implementation in the [Langfuse Example](https://github.com/langfuse/langfuse-examples/tree/main/applications/user-feedback) repository.
+
+### 1. Return trace ID to frontend
+
+Your backend sends the trace ID so frontend can link feedback to the trace.
+
+```typescript
+// app/api/chat/route.ts
+import { getActiveTraceId } from "@Langfuse/tracing";
+
+export const POST = observe(async (req: Request) => {
+ const result = streamText({
+ model: openai('gpt-4o-mini'),
+ messages: convertToModelMessages(messages),
+ });
+ return result.toUIMessageStreamResponse({
+ generateMessageId: () => getActiveTraceId() || "",
+ });
+});
+```
+
+### 2. Collect feedback in frontend
+
+Use Langfuse Web SDK to send feedback as a score.
+
+```typescript
+import { LangfuseWeb } from "langfuse";
+
+const langfuse = new LangfuseWeb({
+ publicKey: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY,
+ baseUrl: process.env.NEXT_PUBLIC_LANGFUSE_HOST,
+});
+function FeedbackButtons({ messageId }: { messageId: string }) {
+ const handleFeedback = (value: number, comment?: string) => {
+ langfuse.score({
+ traceId: messageId,
+ name: "user-feedback",
+ value: value, // 1 for positive, 0 for negative
+ comment: comment,
+ });
+ };
+ return (
+
+
+
+
+ );
+}
+```
+
+### 3. View feedback in Langfuse
+
+Feedback appears as scores on traces. You can filter by `user-feedback < 1` to find low-rated responses.
+
+
+
+## Server-side Feedback
+
+Record feedback from your backend when needed, such as after a user survey or follow-up interaction. You could also use this to log implicit feedback signals such as ticket closures or successful task completions.
+
+```python
+from langfuse import get_client
+langfuse = get_client()
+
+# Check if customer support ticket was resolved successfully
+ticket_status = checkIfTicketClosed(ticket_id="ticket-456")
+if ticket_status.is_closed:
+ langfuse.create_score(
+ trace_id=ticket_status.trace_id,
+ name="ticket-resolution",
+ value=1,
+ comment=f"Ticket closed successfully after {ticket_status.resolution_time}"
+ )
+else:
+ langfuse.create_score(
+ trace_id=ticket_status.trace_id,
+ name="ticket-resolution",
+ value=0,
+ comment=f"Ticket escalated to human agent"
+ )
+```
+
+## Implicit Feedback with LLM-as-a-Judge
+
+Automatically evaluate every response for qualities like user sentiment, satisfaction, or engagement using LLMs as judges. This lets you gather large-scale feedback without user intervention.
+
+
+
+See [LLM-as-a-Judge Evaluators](/docs/evaluation/evaluation-methods/llm-as-a-judge) for implementation patterns and examples.
+
+## Example App
+
+The [user-feedback example](https://github.com/langfuse/langfuse-examples/tree/main/applications/user-feedback) shows a complete Next.js implementation with:
+- OpenTelemetry tracing
+- Thumbs up/down with optional comments
+- Session tracking across conversations
diff --git a/content/academy/what-to-evaluate.mdx b/content/academy/what-to-evaluate.mdx
new file mode 100644
index 000000000..f31aa5a07
--- /dev/null
+++ b/content/academy/what-to-evaluate.mdx
@@ -0,0 +1,171 @@
+---
+title: What to Evaluate
+description: A practical guide to identifying, categorizing, and analyzing failure modes in LLM applications using error analysis.
+---
+
+
+
+To improve your LLM app, you must understand **how it fails**. Aggregate metrics won't tell you if your system retrieves the wrong documents or if the model's tone alienates users. **Error analysis** provides this crucial context.
+
+_The framework in this guide is adapted from Hamel Husain's [Eval FAQ](https://hamel.dev/blog/posts/evals-faq/why-is-error-analysis-so-important-in-llm-evals-and-how-is-it-performed.html)._
+
+---
+
+This guide describes a four-step process to **identify, categorize, and quantify your application's unique failure modes**. The result is a specific evaluation framework that is far more useful than generic metrics:
+
+1. [Gather a diverse dataset of traces](#gather-dataset)
+2. [Open code to surface failure patterns](#open-coding)
+3. [Structure failure modes](#structure-failure-modes)
+4. [Label and quantify](#label-and-quantify)
+
+---
+
+I'll demonstrate this process using an [example chatbot in the Langfuse documentation](/docs/demo) that uses the Vercel AI SDK and has access to a RAG tool to retrieve documents from the Langfuse documentation. The example chat app logs traces into the Langfuse example project and has already answered 19k user queries in the past year.
+
+Here's the chat interface (you can find the example chat app [here](/docs/demo)):
+
+
+ 
+
+
+## 1. Gather a Diverse Dataset [#gather-dataset]
+
+To start our error analysis, we assemble a representative dataset of 50-100 traces produced by the example chat app. The quality of your analysis depends on the diversity of this initial data.
+
+**Existing Production Traces:** If you already have real user traces, as in our example, create your dataset based on them. I recommend first manually clicking through your traces, focusing only on the user input, and adding a diverse set of traces to an annotation queue.
+
+You can also query for traces with negative user feedback, long conversations, high latency, or specific user metadata. The goal is not a random sample, but a set that covers a wide range of user intents and potential edge cases.
+
+In Langfuse, you can bulk add traces to an annotation queue [or a dataset](/changelog/2025-12-11-batch-add-observations-to-dataset) by clicking the "Actions" button:
+
+
+ 
+
+
+**Synthetic Dataset:** If you lack production data, generate a synthetic dataset covering anticipated user behaviors and potential failure points. We have a Python cookbook that shows how to do this [here](/guides/cookbook/example_synthetic_datasets). Once created, add these traces to a Langfuse Annotation Queue. Note that the quality of your dataset matters a lot for the success of your error analysis; it needs to be diverse and representative of the real world.
+
+The [Annotation Queue](/docs/evaluation/evaluation-methods/annotation#annotation-queues) we created will serve as your workspace for the analysis. For our example chatbot, we selected 40 traces reflecting different user questions, from simple definitions to complex comparisons:
+
+
+ 
+
+
+## 2. Open Coding: Surface Failure Patterns [#open-coding]
+
+In the next step, we open our Annotation Queue and carefully review every trace and its associated tool use. The objective is to apply raw, descriptive labels without forcing them into predefined categories.
+
+For each trace, assign two annotations:
+
+- A binary score: Pass or Fail. This forces a clear judgment call.
+
+- A free-text comment: Describe the first point of failure you observe. This process is called [open coding](https://hamel.dev/blog/posts/evals-faq/#open-coding), as we are not forcing any categories on the data.
+
+If you have traces with multiple errors, focusing on the first failure is efficient. A single upstream error, like incorrect document retrieval, often causes multiple downstream issues. Fixing the root cause resolves them all. Your comment should be a raw observation, not a premature diagnosis.
+
+Here are some examples from our example chat app:
+
+import {
+ Carousel,
+ CarouselContent,
+ CarouselItem,
+ CarouselPrevious,
+ CarouselNext,
+} from "@/components/ui/carousel";
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+
+
+## 3. Structure Failure Modes [#structure-failure-modes]
+
+After annotating all traces, the next step is to structure your free-text comments into a coherent taxonomy.
+
+Export your comments from the Langfuse annotation job (you can [query your comments via the Langfuse API](https://colab.research.google.com/drive/1ErETZNWHyOjkG262bZHh-j3Vo9z83qUj)). You can use an LLM to perform an initial clustering of these notes into related themes. Review and manually refine the LLM's output to ensure the categories are distinct, comprehensive, and accurately reflect your application's specific issues.
+
+For our docs chatbot, we used the following prompt on our exported annotations:
+
+> You are given a list of open-ended annotations describing failures of an LLM-powered assistant that answers questions about Langfuse. Organize these into a small set of coherent failure categories, grouping similar mistakes together. For each category, provide a concise descriptive title and a one-line definition. Only cluster based on the issues in the annotations—do not invent new failure types.
+
+This produced a clear taxonomy:
+
+| Failure Mode | Definition |
+| -------------------------------------- | ----------------------------------------------------------------------------------------------- |
+| Hallucinations / Incorrect Information | The assistant gives factually wrong answers or shows lack of knowledge about the domain. |
+| Context Retrieval / RAG Issues | Failures related to retrieving or using the right documents. |
+| Irrelevant or Off-Topic Responses | The assistant produces content unrelated to the user's question. |
+| Generic or Unhelpful Responses | Answers are too broad, vague, or do not directly address the user's question. |
+| Formatting / Presentation Issues | Problems with response delivery, such as missing code blocks or links. |
+| Interaction Style / Missing Follow-ups | The assistant fails to ask clarifying questions or misses opportunities for guided interaction. |
+
+## 4. Label and Quantify [#label-and-quantify]
+
+With our error labels in place, we can now annotate our dataset with these failure modes.
+
+First, create a new Score configuration in Langfuse containing each failure mode as a boolean or categorical option. Then, re-annotate your dataset using this new, structured schema.
+
+This labeled dataset allows you to use Langfuse analytics to pivot and aggregate the data. You can now answer critical questions like, "What is our most frequent failure mode?" For our example chatbot, the analysis revealed that Context Retrieval Issues were the most common problem.
+
+Here are the results after labeling our dataset:
+
+
+ 
+
+
+## Common Pitfalls
+
+- **Generic Metrics:** Avoid starting with off-the-shelf metrics like "conciseness" or "hallucinations." Let your application's actual failures define your evaluation criteria.
+
+- **One-and-Done Analysis:** Error analysis is not a static task. As your application and user behavior evolve, so will its failure modes. Make this process a recurring part of your development cycle.
+
+## Next Steps
+
+This error analysis produces a quantified, application-specific understanding of your primary issues. These insights provide a clear roadmap for targeted improvements, whether in your prompts, RAG pipeline, or model selection.
+
+The structured failure modes you defined serve as the foundation for building automated evaluators, which can scale this analysis across your application. However, before setting up automated evaluators, ensure you first address the obvious issues encountered during the error analysis. You can typically go through multiple rounds of this process before reaching a plateau.
+
+In the next blog post, we will set up automated evaluators and use them to continuously improve our example chatbot:
+
+import { FileCode } from "lucide-react";
+
+
+ } arrow />
+
diff --git a/content/academy/what-to-monitor.mdx b/content/academy/what-to-monitor.mdx
new file mode 100644
index 000000000..7b469f029
--- /dev/null
+++ b/content/academy/what-to-monitor.mdx
@@ -0,0 +1,4 @@
+---
+title: What to Monitor
+description: "Learn how to use chatbot analytics tools to monitor performance, track user interactions, and improve your AI chatbot. Covers key metrics, evaluation strategies, and analytics platforms."
+---
diff --git a/content/faq/all/what-does-a-good-trace-look-like.mdx b/content/faq/all/what-does-a-good-trace-look-like.mdx
index ef0c30623..98297da0b 100644
--- a/content/faq/all/what-does-a-good-trace-look-like.mdx
+++ b/content/faq/all/what-does-a-good-trace-look-like.mdx
@@ -49,7 +49,9 @@ Framework integrations typically set these types automatically. If you're instru
Not every observation in the tree is useful for understanding what your application did. HTTP spans, database queries, and framework internals often add clutter without giving you meaningful insight. If you see observations like these polluting your trace tree, you can [filter them out](/faq/all/unwanted-http-database-spans).
+
+