From 70b54df61f704116a8cc249304e0798eb5b82532 Mon Sep 17 00:00:00 2001 From: Isha Zaka Date: Wed, 7 Oct 2026 18:42:03 -0400 Subject: [PATCH] feat(eval): add Ragas-style context precision and recall metrics --- src/dynavec/eval/__init__.py | 14 +- src/dynavec/eval/base.py | 43 ++++++ src/dynavec/eval/judges.py | 24 +++ src/dynavec/eval/metrics.py | 291 ++++++++++++++++++++++++++++++++++- src/dynavec/eval/runner.py | 59 ++++++- tests/test_eval.py | 254 ++++++++++++++++++++++++++++++ 6 files changed, 679 insertions(+), 6 deletions(-) diff --git a/src/dynavec/eval/__init__.py b/src/dynavec/eval/__init__.py index fcf201e..700d796 100644 --- a/src/dynavec/eval/__init__.py +++ b/src/dynavec/eval/__init__.py @@ -1,4 +1,4 @@ -"""Evaluation framework for RAG faithfulness, answer relevance, and retrieval quality (Recall/MRR/nDCG).""" +"""Evaluation framework for RAG generation quality, context quality, and retrieval metrics.""" from __future__ import annotations @@ -6,6 +6,9 @@ AnswerRelevanceResult, BaseJudge, ClaimVerification, + ContextPrecisionResult, + ContextRecallResult, + ContextRelevanceVerification, FaithfulnessResult, RAGEvalResult, extract_json, @@ -13,6 +16,7 @@ from .chart import plot_eval_summary, plot_retrieval_metrics from .judges import ( BedrockJudge, + ChatModelJudge, CustomJudge, GeminiJudge, MockJudge, @@ -20,6 +24,8 @@ ) from .metrics import ( evaluate_answer_relevance, + evaluate_context_precision, + evaluate_context_recall, evaluate_faithfulness, evaluate_rag, ) @@ -70,4 +76,10 @@ "compute_ndcg_at_k", "compute_precision_at_k", "plot_retrieval_metrics", + "ContextRelevanceVerification", + "ContextPrecisionResult", + "ContextRecallResult", + "evaluate_context_precision", + "evaluate_context_recall", + "ChatModelJudge", ] diff --git a/src/dynavec/eval/base.py b/src/dynavec/eval/base.py index 60ac10d..fea3fd5 100644 --- a/src/dynavec/eval/base.py +++ b/src/dynavec/eval/base.py @@ -99,6 +99,46 @@ def to_dict(self) -> dict[str, Any]: return asdict(self) +@dataclass +class ContextRelevanceVerification: + """Relevance judgment for a single retrieved context chunk.""" + + context: str + relevant: bool + reasoning: str = "" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class ContextPrecisionResult: + """Result of a context precision evaluation.""" + + score: float + contexts: list[ContextRelevanceVerification] = field(default_factory=list) + relevant_count: int = 0 + total_count: int = 0 + reasoning: str = "" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass +class ContextRecallResult: + """Result of a context recall evaluation.""" + + score: float + claims: list[ClaimVerification] = field(default_factory=list) + supported_count: int = 0 + total_count: int = 0 + reasoning: str = "" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @dataclass class RAGEvalResult: """Combined RAG evaluation result for a query, context, and answer.""" @@ -109,6 +149,9 @@ class RAGEvalResult: faithfulness: FaithfulnessResult | None = None answer_relevance: AnswerRelevanceResult | None = None latency_ms: float = 0.0 + reference_answer: str | None = None + context_precision: ContextPrecisionResult | None = None + context_recall: ContextRecallResult | None = None def to_dict(self) -> dict[str, Any]: return asdict(self) diff --git a/src/dynavec/eval/judges.py b/src/dynavec/eval/judges.py index 4a59501..6847dbb 100644 --- a/src/dynavec/eval/judges.py +++ b/src/dynavec/eval/judges.py @@ -6,11 +6,35 @@ from collections.abc import Callable from typing import Any +from ..chat.base import ChatModel, Message from ..credentials import resolve_session from ..exceptions import MissingDependencyError from .base import BaseJudge +class ChatModelJudge(BaseJudge): + """LLM judge backed by Dynavec's ChatModel abstraction. + + This allows any chat model implementing the Dynavec model layer to be used + for evaluation without coupling metrics to a specific provider SDK. + """ + + def __init__( + self, + model: ChatModel, + **invoke_kwargs: Any, + ) -> None: + self.model = model + self.invoke_kwargs = invoke_kwargs + + def judge(self, prompt: str) -> str: + result = self.model.invoke( + [Message(role="user", content=prompt)], + **self.invoke_kwargs, + ) + return result.message.content or "" + + class OpenAIJudge(BaseJudge): """LLM judge using OpenAI's chat completions API (bring your own API key). diff --git a/src/dynavec/eval/metrics.py b/src/dynavec/eval/metrics.py index d93da18..0cd4506 100644 --- a/src/dynavec/eval/metrics.py +++ b/src/dynavec/eval/metrics.py @@ -1,4 +1,4 @@ -"""Core RAG evaluation metrics: Faithfulness (hallucination detection) and Answer Relevance.""" +"""Core RAG evaluation metrics: faithfulness, answer relevance, context precision, and context recall.""" from __future__ import annotations @@ -9,6 +9,9 @@ AnswerRelevanceResult, BaseJudge, ClaimVerification, + ContextPrecisionResult, + ContextRecallResult, + ContextRelevanceVerification, FaithfulnessResult, RAGEvalResult, ) @@ -65,6 +68,70 @@ """ +_CONTEXT_PRECISION_PROMPT_TEMPLATE = """You are an expert evaluator assessing CONTEXT PRECISION for a RAG system. + +Task: +Evaluate each retrieved context chunk independently and determine whether it is useful for answering the user query according to the reference answer. + +A context is relevant when it contains information that helps support or produce the reference answer. +A context is irrelevant when it does not help answer the query or contains unrelated information. + +Preserve the original context rank. + +User Query: +{query} + +Reference Answer: +{reference_answer} + +Retrieved Contexts: +{contexts_text} + +Respond ONLY with a JSON object in this exact schema: +{{ + "contexts": [ + {{ + "index": 1, + "relevant": true, + "reasoning": "" + }} + ], + "reasoning": "" +}} +""" + + +_CONTEXT_RECALL_PROMPT_TEMPLATE = """You are an expert evaluator assessing CONTEXT RECALL for a RAG system. + +Task: +1. Break the reference answer into discrete, atomic factual claims. +2. For each claim, determine whether it is supported by the retrieved context. +3. A claim is supported when the retrieved context contains enough information to establish that claim. +4. Mark unsupported claims when the required information is missing from the retrieved context. + +Retrieved Context: +{context_text} + +User Query: +{query} + +Reference Answer: +{reference_answer} + +Respond ONLY with a JSON object in this exact schema: +{{ + "claims": [ + {{ + "claim": "", + "supported": true, + "reasoning": "" + }} + ], + "reasoning": "" +}} +""" + + def evaluate_faithfulness( query: str, context: str | Sequence[str], @@ -214,6 +281,190 @@ def evaluate_answer_relevance( ) +def evaluate_context_precision( + query: str, + context: str | Sequence[str], + reference_answer: str, + judge: BaseJudge, +) -> ContextPrecisionResult: + """Evaluate whether relevant retrieved contexts are ranked ahead of irrelevant ones.""" + + if isinstance(context, str): + contexts = [context.strip()] if context.strip() else [] + else: + contexts = [chunk.strip() for chunk in context if chunk.strip()] + + if not contexts: + return ContextPrecisionResult( + score=0.0, + contexts=[], + relevant_count=0, + total_count=0, + reasoning="Empty context provided.", + ) + + if not reference_answer or not reference_answer.strip(): + return ContextPrecisionResult( + score=0.0, + contexts=[ + ContextRelevanceVerification( + context=chunk, + relevant=False, + reasoning="No reference answer provided.", + ) + for chunk in contexts + ], + relevant_count=0, + total_count=len(contexts), + reasoning="Empty reference answer provided.", + ) + + contexts_text = "\n\n".join( + f"[{index}] {chunk}" for index, chunk in enumerate(contexts, start=1) + ) + + prompt = _CONTEXT_PRECISION_PROMPT_TEMPLATE.format( + query=query.strip() or "N/A", + reference_answer=reference_answer.strip(), + contexts_text=contexts_text, + ) + + data = judge.judge_structured(prompt) + raw_contexts = data.get("contexts", []) + reasoning = str(data.get("reasoning", "")) + + verdicts_by_index: dict[int, dict[str, object]] = {} + + if isinstance(raw_contexts, list): + for position, item in enumerate(raw_contexts, start=1): + if not isinstance(item, dict): + continue + + try: + index = int(item.get("index", position)) + except (TypeError, ValueError): + index = position + + if 1 <= index <= len(contexts): + verdicts_by_index[index] = item + + verifications: list[ContextRelevanceVerification] = [] + relevant_count = 0 + precision_sum = 0.0 + + for rank, chunk in enumerate(contexts, start=1): + item = verdicts_by_index.get(rank, {}) + relevant = item.get("relevant", False) is True + chunk_reasoning = str(item.get("reasoning", "")).strip() + + if relevant: + relevant_count += 1 + precision_sum += relevant_count / rank + + verifications.append( + ContextRelevanceVerification( + context=chunk, + relevant=relevant, + reasoning=chunk_reasoning, + ) + ) + + score = precision_sum / relevant_count if relevant_count > 0 else 0.0 + + return ContextPrecisionResult( + score=round(score, 4), + contexts=verifications, + relevant_count=relevant_count, + total_count=len(contexts), + reasoning=reasoning, + ) + + +def evaluate_context_recall( + query: str, + context: str | Sequence[str], + reference_answer: str, + judge: BaseJudge, +) -> ContextRecallResult: + """Evaluate how much of the reference answer is supported by retrieved context.""" + + if not reference_answer or not reference_answer.strip(): + return ContextRecallResult( + score=0.0, + claims=[], + supported_count=0, + total_count=0, + reasoning="Empty reference answer provided.", + ) + + if isinstance(context, str): + context_text = context.strip() + else: + context_text = "\n\n---\n\n".join(chunk.strip() for chunk in context if chunk.strip()) + + if not context_text: + return ContextRecallResult( + score=0.0, + claims=[ + ClaimVerification( + claim=reference_answer.strip(), + supported=False, + reasoning="No context provided to support the reference answer.", + ) + ], + supported_count=0, + total_count=1, + reasoning="Empty context provided.", + ) + + prompt = _CONTEXT_RECALL_PROMPT_TEMPLATE.format( + context_text=context_text, + query=query.strip() or "N/A", + reference_answer=reference_answer.strip(), + ) + + data = judge.judge_structured(prompt) + raw_claims = data.get("claims", []) + reasoning = str(data.get("reasoning", "")) + + verifications: list[ClaimVerification] = [] + supported_count = 0 + + if isinstance(raw_claims, list): + for item in raw_claims: + if not isinstance(item, dict): + continue + + claim = str(item.get("claim", "")).strip() + supported = item.get("supported", False) is True + claim_reasoning = str(item.get("reasoning", "")).strip() + + if not claim: + continue + + if supported: + supported_count += 1 + + verifications.append( + ClaimVerification( + claim=claim, + supported=supported, + reasoning=claim_reasoning, + ) + ) + + total_count = len(verifications) + score = supported_count / total_count if total_count > 0 else 0.0 + + return ContextRecallResult( + score=round(score, 4), + claims=verifications, + supported_count=supported_count, + total_count=total_count, + reasoning=reasoning, + ) + + def evaluate_rag( query: str, context: str | Sequence[str], @@ -221,8 +472,11 @@ def evaluate_rag( judge: BaseJudge, run_faithfulness: bool = True, run_answer_relevance: bool = True, + reference_answer: str | None = None, + run_context_precision: bool = True, + run_context_recall: bool = True, ) -> RAGEvalResult: - """Perform combined RAG evaluation (Faithfulness + Answer Relevance). + """Perform combined RAG evaluation across generation and context quality metrics. Parameters ---------- @@ -238,6 +492,12 @@ def evaluate_rag( Whether to evaluate faithfulness (default True). run_answer_relevance: Whether to evaluate answer relevance (default True). + reference_answer: + Optional labeled/reference answer required for context metrics. + run_context_precision: + Whether to evaluate context precision when a reference answer is provided. + run_context_recall: + Whether to evaluate context recall when a reference answer is provided. Returns ------- @@ -259,6 +519,30 @@ def evaluate_rag( else None ) + has_reference = bool(reference_answer and reference_answer.strip()) + + context_precision_res = ( + evaluate_context_precision( + query=query, + context=context, + reference_answer=reference_answer or "", + judge=judge, + ) + if run_context_precision and has_reference + else None + ) + + context_recall_res = ( + evaluate_context_recall( + query=query, + context=context, + reference_answer=reference_answer or "", + judge=judge, + ) + if run_context_recall and has_reference + else None + ) + elapsed_ms = (time.perf_counter() - t0) * 1000.0 return RAGEvalResult( @@ -268,4 +552,7 @@ def evaluate_rag( faithfulness=faithfulness_res, answer_relevance=relevance_res, latency_ms=round(elapsed_ms, 2), + reference_answer=reference_answer, + context_precision=context_precision_res, + context_recall=context_recall_res, ) diff --git a/src/dynavec/eval/runner.py b/src/dynavec/eval/runner.py index 3bb083b..5895d65 100644 --- a/src/dynavec/eval/runner.py +++ b/src/dynavec/eval/runner.py @@ -23,6 +23,8 @@ class EvalSummary: pass_threshold: float p95_latency_ms: float results: list[RAGEvalResult] = field(default_factory=list) + mean_context_precision: float = 0.0 + mean_context_recall: float = 0.0 def to_dict(self) -> dict[str, Any]: data = asdict(self) @@ -52,6 +54,9 @@ def evaluate_sample( answer: str, run_faithfulness: bool = True, run_answer_relevance: bool = True, + reference_answer: str | None = None, + run_context_precision: bool = True, + run_context_recall: bool = True, ) -> RAGEvalResult: """Evaluate a single query-context-answer triplet.""" return evaluate_rag( @@ -61,6 +66,9 @@ def evaluate_sample( judge=self.judge, run_faithfulness=run_faithfulness, run_answer_relevance=run_answer_relevance, + reference_answer=reference_answer, + run_context_precision=run_context_precision, + run_context_recall=run_context_recall, ) def run( @@ -68,24 +76,37 @@ def run( dataset: Sequence[dict[str, Any] | EvalTriplet], run_faithfulness: bool = True, run_answer_relevance: bool = True, + run_context_precision: bool = True, + run_context_recall: bool = True, ) -> EvalSummary: """Evaluate an entire dataset in batch and compute aggregate quality metrics. Each item in `dataset` can be: - - A dict with keys ``"query"``, ``"context"``, ``"answer"`` - - A 3-tuple ``(query, context, answer)`` + - A dict with ``query``, ``context``, ``answer`` and optional ``reference_answer`` + - A legacy 3-tuple ``(query, context, answer)`` + + Context precision and context recall run only when a reference answer is present. """ results: list[RAGEvalResult] = [] faithfulness_scores: list[float] = [] relevance_scores: list[float] = [] + context_precision_scores: list[float] = [] + context_recall_scores: list[float] = [] latencies: list[float] = [] passed_count = 0 for item in dataset: + reference_answer: str | None = None + if isinstance(item, dict): q = str(item.get("query", "")) c = item.get("context", []) a = str(item.get("answer", "")) + + raw_reference = item.get("reference_answer") + if raw_reference is not None: + reference_answer = str(raw_reference) + elif isinstance(item, (tuple, list)) and len(item) >= 3: q, c, a = item[0], item[1], item[2] else: @@ -97,21 +118,41 @@ def run( answer=a, run_faithfulness=run_faithfulness, run_answer_relevance=run_answer_relevance, + reference_answer=reference_answer, + run_context_precision=run_context_precision, + run_context_recall=run_context_recall, ) results.append(res) latencies.append(res.latency_ms) is_pass = True + if res.faithfulness is not None: faithfulness_scores.append(res.faithfulness.score) if res.faithfulness.score < self.pass_threshold: is_pass = False + if res.answer_relevance is not None: relevance_scores.append(res.answer_relevance.score) if res.answer_relevance.score < self.pass_threshold: is_pass = False - if is_pass and (res.faithfulness is not None or res.answer_relevance is not None): + if res.context_precision is not None: + context_precision_scores.append(res.context_precision.score) + if res.context_precision.score < self.pass_threshold: + is_pass = False + + if res.context_recall is not None: + context_recall_scores.append(res.context_recall.score) + if res.context_recall.score < self.pass_threshold: + is_pass = False + + if is_pass and ( + res.faithfulness is not None + or res.answer_relevance is not None + or res.context_precision is not None + or res.context_recall is not None + ): passed_count += 1 total = len(results) @@ -121,6 +162,16 @@ def run( else 0.0 ) mean_r = float(sum(relevance_scores) / len(relevance_scores)) if relevance_scores else 0.0 + mean_cp = ( + float(sum(context_precision_scores) / len(context_precision_scores)) + if context_precision_scores + else 0.0 + ) + mean_cr = ( + float(sum(context_recall_scores) / len(context_recall_scores)) + if context_recall_scores + else 0.0 + ) pass_rate = float(passed_count / total) if total > 0 else 0.0 # Calculate p95 latency @@ -138,4 +189,6 @@ def run( pass_threshold=self.pass_threshold, p95_latency_ms=round(p95_lat, 2), results=results, + mean_context_precision=round(mean_cp, 4), + mean_context_recall=round(mean_cr, 4), ) diff --git a/tests/test_eval.py b/tests/test_eval.py index 621c0d3..dc51834 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -8,9 +8,13 @@ import pytest +from dynavec.chat.base import ChatModel, ChatResult, Message from dynavec.dashboard import _make_handler from dynavec.eval import ( AnswerRelevanceResult, + ChatModelJudge, + ContextPrecisionResult, + ContextRecallResult, CustomJudge, EvalRunner, EvalSummary, @@ -20,6 +24,8 @@ OpenAIJudge, RAGEvalResult, evaluate_answer_relevance, + evaluate_context_precision, + evaluate_context_recall, evaluate_faithfulness, evaluate_rag, extract_json, @@ -68,6 +74,27 @@ def test_invalid_json_raises_value_error(self) -> None: class TestJudges: + def test_chat_model_judge_uses_model_layer(self) -> None: + model = MagicMock(spec=ChatModel) + model.invoke.return_value = ChatResult( + message=Message( + role="assistant", + content='{"score": 0.91, "reasoning": "Model-layer judge"}', + ) + ) + + judge = ChatModelJudge(model) + result = judge.judge_structured("evaluate this") + + assert result["score"] == 0.91 + + model.invoke.assert_called_once() + + messages = model.invoke.call_args.args[0] + assert len(messages) == 1 + assert messages[0].role == "user" + assert messages[0].content == "evaluate this" + def test_mock_judge_sequential_and_default(self) -> None: resp1 = {"claims": [{"claim": "c1", "supported": True}], "reasoning": "ok"} resp2 = {"score": 0.9, "reasoning": "relevant"} @@ -289,7 +316,164 @@ def test_empty_query_or_answer(self) -> None: # =========================================================================== +class TestContextPrecisionMetric: + def test_rank_sensitive_context_precision(self) -> None: + judge = MockJudge( + default_response={ + "contexts": [ + { + "index": 1, + "relevant": True, + "reasoning": "Useful context.", + }, + { + "index": 2, + "relevant": False, + "reasoning": "Irrelevant context.", + }, + { + "index": 3, + "relevant": True, + "reasoning": "Useful context.", + }, + ], + "reasoning": "Two of three contexts are relevant.", + } + ) + + result = evaluate_context_precision( + query="Where is the Eiffel Tower?", + context=[ + "The Eiffel Tower is located in Paris.", + "Python is a programming language.", + "Paris is the capital of France.", + ], + reference_answer="The Eiffel Tower is located in Paris, France.", + judge=judge, + ) + + assert isinstance(result, ContextPrecisionResult) + assert result.score == pytest.approx(0.8333) + assert result.relevant_count == 2 + assert result.total_count == 3 + assert result.contexts[0].relevant is True + assert result.contexts[1].relevant is False + assert result.contexts[2].relevant is True + + +class TestContextRecallMetric: + def test_partial_context_recall(self) -> None: + judge = MockJudge( + default_response={ + "claims": [ + { + "claim": "The Eiffel Tower is located in Paris.", + "supported": True, + "reasoning": "The retrieved context states this directly.", + }, + { + "claim": "The Eiffel Tower was completed in 1889.", + "supported": True, + "reasoning": "The completion year appears in context.", + }, + { + "claim": "The Eiffel Tower was designed by Gustave Eiffel.", + "supported": False, + "reasoning": "The designer is not mentioned.", + }, + ], + "reasoning": "Two of three reference claims are supported.", + } + ) + + result = evaluate_context_recall( + query="Tell me about the Eiffel Tower.", + context=[ + "The Eiffel Tower is located in Paris.", + "Construction was completed in 1889.", + ], + reference_answer=( + "The Eiffel Tower is located in Paris. " + "It was completed in 1889 and was designed by Gustave Eiffel." + ), + judge=judge, + ) + + assert isinstance(result, ContextRecallResult) + assert result.score == pytest.approx(0.6667) + assert result.supported_count == 2 + assert result.total_count == 3 + assert result.claims[0].supported is True + assert result.claims[2].supported is False + + class TestEvalRunner: + def test_eval_runner_batch_context_metrics(self) -> None: + judge = MockJudge( + responses=[ + # Sample 1 context precision + { + "contexts": [ + {"index": 1, "relevant": True}, + {"index": 2, "relevant": False}, + ] + }, + # Sample 1 context recall + { + "claims": [ + {"claim": "claim 1", "supported": True}, + {"claim": "claim 2", "supported": True}, + ] + }, + # Sample 2 context precision + { + "contexts": [ + {"index": 1, "relevant": False}, + {"index": 2, "relevant": True}, + ] + }, + # Sample 2 context recall + { + "claims": [ + {"claim": "claim 1", "supported": True}, + {"claim": "claim 2", "supported": False}, + ] + }, + ] + ) + + runner = EvalRunner(judge=judge, pass_threshold=0.7) + + dataset = [ + { + "query": "Q1", + "context": ["C1 relevant", "C1 irrelevant"], + "answer": "A1", + "reference_answer": "Reference 1", + }, + { + "query": "Q2", + "context": ["C2 irrelevant", "C2 relevant"], + "answer": "A2", + "reference_answer": "Reference 2", + }, + ] + + summary = runner.run( + dataset, + run_faithfulness=False, + run_answer_relevance=False, + ) + + assert summary.total_samples == 2 + assert summary.mean_context_precision == pytest.approx(0.75) + assert summary.mean_context_recall == pytest.approx(0.75) + assert summary.pass_rate == 0.5 + + assert summary.results[0].reference_answer == "Reference 1" + assert summary.results[0].context_precision is not None + assert summary.results[0].context_recall is not None + def test_evaluate_rag_combined(self) -> None: f_resp = { "claims": [{"claim": "Fact 1", "supported": True}], @@ -316,6 +500,76 @@ def test_evaluate_rag_combined(self) -> None: assert d["query"] == "What is dynavec?" assert d["faithfulness"]["score"] == 1.0 + def test_evaluate_rag_all_four_metrics(self) -> None: + faithfulness_response = { + "claims": [ + { + "claim": "The Eiffel Tower is in Paris.", + "supported": True, + } + ] + } + + relevance_response = { + "score": 0.9, + "reasoning": "Direct answer.", + } + + precision_response = { + "contexts": [ + {"index": 1, "relevant": True}, + {"index": 2, "relevant": False}, + ] + } + + recall_response = { + "claims": [ + { + "claim": "The Eiffel Tower is in Paris.", + "supported": True, + }, + { + "claim": "It was completed in 1889.", + "supported": False, + }, + ] + } + + judge = MockJudge( + responses=[ + faithfulness_response, + relevance_response, + precision_response, + recall_response, + ] + ) + + result = evaluate_rag( + query="Where is the Eiffel Tower?", + context=[ + "The Eiffel Tower is located in Paris.", + "Python is a programming language.", + ], + answer="The Eiffel Tower is in Paris.", + reference_answer=("The Eiffel Tower is in Paris and was completed in 1889."), + judge=judge, + ) + + assert result.faithfulness is not None + assert result.faithfulness.score == 1.0 + + assert result.answer_relevance is not None + assert result.answer_relevance.score == 0.9 + + assert result.context_precision is not None + assert result.context_precision.score == 1.0 + + assert result.context_recall is not None + assert result.context_recall.score == 0.5 + + assert result.reference_answer is not None + assert len(judge.call_history) == 4 + def test_eval_runner_batch_dataset(self) -> None: # Sample 1: Faithful (1.0) & Relevant (0.9) -> Pass # Sample 2: Partial (0.5) & Relevant (0.8) -> Fail (since pass_threshold=0.7)