feat(eval): add Ragas-style context precision and recall metrics

src/dynavec/eval/__init__.py

@@ -1,25 +1,31 @@
-"""Evaluation framework for RAG faithfulness, answer relevance, and retrieval quality (Recall/MRR/nDCG)."""
+"""Evaluation framework for RAG generation quality, context quality, and retrieval metrics."""
 
 from __future__ import annotations
 
 from .base import (
     AnswerRelevanceResult,
     BaseJudge,
     ClaimVerification,
+    ContextPrecisionResult,
+    ContextRecallResult,
+    ContextRelevanceVerification,
     FaithfulnessResult,
     RAGEvalResult,
     extract_json,
 )
 from .chart import plot_eval_summary, plot_retrieval_metrics
 from .judges import (
     BedrockJudge,
+    ChatModelJudge,
     CustomJudge,
     GeminiJudge,
     MockJudge,
     OpenAIJudge,
 )
 from .metrics import (
     evaluate_answer_relevance,
+    evaluate_context_precision,
+    evaluate_context_recall,
     evaluate_faithfulness,
     evaluate_rag,
 )
@@ -70,4 +76,10 @@
     "compute_ndcg_at_k",
     "compute_precision_at_k",
     "plot_retrieval_metrics",
+    "ContextRelevanceVerification",
+    "ContextPrecisionResult",
+    "ContextRecallResult",
+    "evaluate_context_precision",
+    "evaluate_context_recall",
+    "ChatModelJudge",
 ]

src/dynavec/eval/base.py

@@ -99,6 +99,46 @@ def to_dict(self) -> dict[str, Any]:
         return asdict(self)
 
 
+@dataclass
+class ContextRelevanceVerification:
+    """Relevance judgment for a single retrieved context chunk."""
+
+    context: str
+    relevant: bool
+    reasoning: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class ContextPrecisionResult:
+    """Result of a context precision evaluation."""
+
+    score: float
+    contexts: list[ContextRelevanceVerification] = field(default_factory=list)
+    relevant_count: int = 0
+    total_count: int = 0
+    reasoning: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class ContextRecallResult:
+    """Result of a context recall evaluation."""
+
+    score: float
+    claims: list[ClaimVerification] = field(default_factory=list)
+    supported_count: int = 0
+    total_count: int = 0
+    reasoning: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
 @dataclass
 class RAGEvalResult:
     """Combined RAG evaluation result for a query, context, and answer."""
@@ -109,6 +149,9 @@ class RAGEvalResult:
     faithfulness: FaithfulnessResult | None = None
     answer_relevance: AnswerRelevanceResult | None = None
     latency_ms: float = 0.0
+    reference_answer: str | None = None
+    context_precision: ContextPrecisionResult | None = None
+    context_recall: ContextRecallResult | None = None
 
     def to_dict(self) -> dict[str, Any]:
         return asdict(self)

src/dynavec/eval/judges.py

@@ -6,11 +6,35 @@
 from collections.abc import Callable
 from typing import Any
 
+from ..chat.base import ChatModel, Message
 from ..credentials import resolve_session
 from ..exceptions import MissingDependencyError
 from .base import BaseJudge
 
 
+class ChatModelJudge(BaseJudge):
+    """LLM judge backed by Dynavec's ChatModel abstraction.
+
+    This allows any chat model implementing the Dynavec model layer to be used
+    for evaluation without coupling metrics to a specific provider SDK.
+    """
+
+    def __init__(
+        self,
+        model: ChatModel,
+        **invoke_kwargs: Any,
+    ) -> None:
+        self.model = model
+        self.invoke_kwargs = invoke_kwargs
+
+    def judge(self, prompt: str) -> str:
+        result = self.model.invoke(
+            [Message(role="user", content=prompt)],
+            **self.invoke_kwargs,
+        )
+        return result.message.content or ""
+
+
 class OpenAIJudge(BaseJudge):
     """LLM judge using OpenAI's chat completions API (bring your own API key).