RAG Evaluation Metrics: RAGAS vs TruLens vs DeepEval (2026)

from ragas.testset.evolutions import simple, reasoning, multi_context
from ragas.testset.generator import TestsetGenerator
from langchain_community.embeddings import OpenAIEmbeddings
from langchain_openai import ChatOpenAI

# Load your documents
from langchain_community.document_loaders import DirectoryLoader
loader = DirectoryLoader("./docs/")
documents = loader.load()

# Configure generator
generator = TestsetGenerator(
    generator_llm=ChatOpenAI(model="gpt-4o"),
    critic_llm=ChatOpenAI(model="gpt-4o"),
    embeddings=OpenAIEmbeddings(),
)

# Generate synthetic test set with different evolution types
testset = generator.generate_with_langchain_docs(
    documents,
    test_size=50,
    distributions={simple: 0.3, reasoning: 0.4, multi_context: 0.3}
)

from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    context_precision,
    context_recall,
    answer_correctness,
)
from ragas import evaluate

result = evaluate(
    dataset=test_dataset,  # question, answer, contexts, ground_truth
    metrics=[
        faithfulness,
        answer_relevancy,
        context_precision,
        context_recall,
        answer_correctness,
    ],
)

print(result)
# {
#   "faithfulness": 0.85,
#   "answer_relevancy": 0.92,
#   "context_precision": 0.78,
#   "context_recall": 0.71,
#   "answer_correctness": 0.83,
# }

from trulens_eval import Feedback, Tru
from trulens_eval.feedback.provider import OpenAI
from trulens_eval.app import App

provider = OpenAI(model_engine="gpt-4o")

# Define feedback functions
f_faithfulness = Feedback(
    provider.faithfulness_with_cot_reasons,
    name="Faithfulness"
).on_input_output()

f_answer_relevance = Feedback(
    provider.relevance_with_cot_reasons,
    name="Answer Relevance"
).on_input_output()

f_context_relevance = Feedback(
    provider.context_relevance_with_cot_reasons,
    name="Context Relevance"
).on_input()

# Wrap your RAG app
from trulens_eval import TruChain
tru_recorder = TruChain(
    chain,
    app_id="my-rag-app-v1",
    feedbacks=[f_faithfulness, f_answer_relevance, f_context_relevance]
)

# Run with monitoring
with tru_recorder as recording:
    chain.invoke({"question": "What is RAG?"})

# View dashboard
Tru().run_dashboard()  # Local dashboard at localhost:8501

import pytest
from deepeval import assert_test
from deepeval.metrics import (
    FaithfulnessMetric,
    AnswerRelevancyMetric,
    ContextRecallMetric,
    HallucinationMetric,
)
from deepeval.test_case import LLMTestCase

# Define test case
test_case = LLMTestCase(
    input="What is the capital of France?",
    actual_output="The capital of France is Paris.",
    retrieval_context=[
        "France is a country in Western Europe.",
        "Paris is the capital of France.",
    ],
    context=["Paris is the capital of France."]  # expected retrieved
)

# Test with multiple metrics
def test_rag_faithfulness():
    assert_test(
        test_case,
        [
            FaithfulnessMetric(threshold=0.7),
            AnswerRelevancyMetric(threshold=0.8),
            ContextRecallMetric(threshold=0.8),
        ]
    )

# Run like normal pytest
# $ pytest test_rag.py -v

from deepeval import evaluate
from deepeval.metrics import GEval, FaithfulnessMetric
from deepeval.test_case import LLMTestCaseParams

# Create multiple test cases
test_cases = [
    LLMTestCase(input="Q1", actual_output="A1", retrieval_context=["..."]),
    LLMTestCase(input="Q2", actual_output="A2", retrieval_context=["..."]),
    # ... 50+ test cases
]

# Evaluate and get structured results
results = evaluate(
    test_cases,
    metrics=[
        FaithfulnessMetric(),
        GEval(
            name="Correctness",
            criteria="Determine if the output is factually correct",
            evaluation_params=[
                LLMTestCaseParams.INPUT,
                LLMTestCaseParams.ACTUAL_OUTPUT,
            ],
        ),
    ],
)

# Results include pass/fail, score, reasons
for result in results.test_results:
    print(f"Passed: {result.is_successful}, Score: {result.metrics_data[0].score}")

1. Use RAGAS to generate 50 synthetic test cases from your documents
2. Use DeepEval to evaluate those tests in a pytest CI pipeline
3. Monitor faithfulness and context recall as your primary metrics
4. Set thresholds: faithfulness ≥ 0.8, context recall ≥ 0.7
5. Run on every PR that changes your RAG pipeline

┌─────────────────────────────────────────────────────────┐
│                   Evaluation Pipeline                     │
├─────────────────────────────────────────────────────────┤
│                                                          │
│  Pre-deployment (CI/CD)       Post-deployment (Monitor)  │
│  ┌──────────────────┐        ┌──────────────────┐       │
│  │ RAGAS: 200 tests │        │ TruLens: Every    │       │
│  │ DeepEval: assert │        │ production query  │       │
│  │ on faithfulness  │        │ Faithfulness +    │       │
│  │ context_recall   │        │ latency tracking  │       │
│  │                  │        │ Dashboard alerts  │       │
│  └──────┬───────────┘        │ on regression    │       │
│         │                    └──────────────────┘       │
│         ▼                                                │
│  ┌──────────────────┐                                    │
│  │ If score drops >  │                                    │
│  │ 5% from baseline: │                                    │
│  │ Block deploy      │                                    │
│  └──────────────────┘                                    │
│                                                          │
│  Weekly: Rerun full RAGAS suite (200 tests)              │
│          Track metric drift over time                    │
│          Update baseline every 2 weeks                   │
│                                                          │
└─────────────────────────────────────────────────────────┘