diff --git a/docs/architecture/platform.md b/docs/architecture/platform.md index a0b03b2..7b73c56 100644 --- a/docs/architecture/platform.md +++ b/docs/architecture/platform.md @@ -818,13 +818,35 @@ Results are normalized into `SearchHit`: - source/index generation - modality - start/end time -- score and score semantics +- `rank`, `score`, and `raw_distance` (see ranking values below) - displayable text/metadata - optional preview reference The application owns limits, score normalization, multimodal fusion and deterministic ordering. UI/API/MCP may request stricter limits but cannot loosen policy. +### Ranking values + +Every search response is self-describing about its numbers so callers never have to +guess which value they received or which direction ranks better. A `RetrievalScoring` +descriptor is attached to both `SearchResult` and `FusedSearchResult`, and the same +descriptor is returned identically across the CLI, HTTP, MCP, stored job results, and +evidence artifacts. + +- `raw_distance` is the raw vector-store distance under `scoring.distance_metric` + (`l2`, `cosine`, or `ip`); smaller is closer. +- `score` on a hit is derived as `score = -raw_distance` (`score_transform`), so larger + ranks better. +- A `SearchHit.rank` is 1-based within one modality channel; ranks from different + channels are not comparable. +- A `FusedMoment.rank`/`score` is the combined position and reciprocal-rank-fusion score + across channels; `fusion.searched_modalities` lists the channels that ran and + `FusedMoment.modalities` lists the channels that contributed to that moment. +- All of these are `ordering_only` (`scoring.score_calibration` and + `fusion.score_calibration`): valid for sorting within a single response, never a + probability or confidence. Calibrated scores are deferred to the end-to-end ranking + evaluation in issue #76. + Actor cluster and detection queries use the same pagination conventions. ## 16. Natural-language query layer diff --git a/src/vidxp/application_models.py b/src/vidxp/application_models.py index 395bde4..cffe6a5 100644 --- a/src/vidxp/application_models.py +++ b/src/vidxp/application_models.py @@ -809,6 +809,61 @@ class FusionProfile(StrEnum): reciprocal_rank = "rrf_v1" +class RankDirection(StrEnum): + """Whether smaller or larger values of a ranking quantity rank better.""" + + lower_is_better = "lower_is_better" + higher_is_better = "higher_is_better" + + +class ScoreCalibration(StrEnum): + """How much meaning a numeric score carries beyond ordering.""" + + ordering_only = "ordering_only" + + +class RetrievalScoring(ApplicationModel): + """Self-describing meaning of the ranking values in a search response. + + Callers see several numbers on a hit (a raw vector-store distance and a + derived score) and a rank. This descriptor states, in the payload itself, + which distance metric produced ``raw_distance``, which direction ranks + better, how ``score`` is derived from the distance, and that neither value + is calibrated. It is returned identically across the CLI, HTTP, MCP, + stored job results, and evidence artifacts so the meaning never drifts + between surfaces. Calibrated, probability-like scores are deferred to the + end-to-end ranking evaluation tracked in issue #76. + """ + + distance_metric: Literal["l2", "cosine", "ip"] = Field( + default="l2", + description="Vector-store distance space that produced each raw_distance.", + ) + raw_distance_direction: Literal[RankDirection.lower_is_better] = Field( + default=RankDirection.lower_is_better, + description="raw_distance sorts ascending; a smaller distance is closer.", + ) + score_transform: Literal["negated_distance"] = Field( + default="negated_distance", + description="Each hit score is derived as score = -raw_distance.", + ) + score_direction: Literal[RankDirection.higher_is_better] = Field( + default=RankDirection.higher_is_better, + description="score sorts descending; a larger score ranks better.", + ) + score_calibration: Literal[ScoreCalibration.ordering_only] = Field( + default=ScoreCalibration.ordering_only, + description=( + "raw_distance and score are ordering_only: valid for sorting hits " + "within one response, never a probability or confidence." + ), + ) + hit_rank_direction: Literal[RankDirection.lower_is_better] = Field( + default=RankDirection.lower_is_better, + description="A hit's per-channel rank starts at 1; rank 1 is the closest.", + ) + + class EvidenceDeliveryMode(StrEnum): none = "none" keyframes = "keyframes" @@ -840,7 +895,13 @@ class EvidenceBoardCandidate(ApplicationModel): representative_timestamp: float = Field(ge=0) frame_index: int | None = Field(default=None, ge=0) frame_match: "EvidenceFrameMatch" - score: float | None = None + score: float | None = Field( + default=None, + description=( + "Combined ordering-only fusion score copied from the source moment; " + "larger ranks better, not a probability." + ), + ) display_text: str | None = Field(default=None, max_length=512) provenance: dict[str, JsonValue] = Field(default_factory=dict) @@ -895,14 +956,32 @@ def _unique_modalities( class SearchHit(ApplicationModel): - rank: int = Field(gt=0) + rank: int = Field( + gt=0, + description=( + "1-based rank of this hit within its own modality channel " + "(rank 1 is closest). Ranks from different channels are not " + "comparable; the combined position is FusedMoment.rank." + ), + ) media_id: MediaId video_id: VideoId generation_id: IndexGenerationId start: float = Field(ge=0) end: float = Field(gt=0) - score: float - raw_distance: float + score: float = Field( + description=( + "Ordering-only channel score, score = -raw_distance, so larger " + "ranks better. Not a probability or confidence; see the response " + "scoring descriptor." + ), + ) + raw_distance: float = Field( + description=( + "Raw vector-store distance under the configured metric; smaller is " + "closer. See scoring.distance_metric for the metric." + ), + ) modality: str = Field(min_length=1) source_id: str = Field(min_length=1) metadata: dict[str, JsonValue] = Field(default_factory=dict) @@ -950,6 +1029,10 @@ class SearchResult(ApplicationModel): query_id: str = Field(min_length=1) query: str = Field(min_length=1) modality: str = Field(min_length=1) + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description="Meaning of the rank, score, and raw_distance on each hit.", + ) hits: tuple[SearchHit, ...] = () def to_dict(self) -> dict[str, Any]: @@ -963,18 +1046,49 @@ class FusionProvenance(ApplicationModel): profile: Literal[FusionProfile.reciprocal_rank] = FusionProfile.reciprocal_rank rank_constant: int = Field(default=60, gt=0) overlap_rule: Literal["connected_intervals"] = "connected_intervals" - requested_modalities: tuple[Identifier, ...] = () - searched_modalities: tuple[Identifier, ...] = () + requested_modalities: tuple[Identifier, ...] = Field( + default=(), + description="Channels the caller asked to search.", + ) + searched_modalities: tuple[Identifier, ...] = Field( + default=(), + description=( + "Channels actually run for this response. A moment's contributing " + "channels are the subset in FusedMoment.modalities." + ), + ) + score_direction: Literal[RankDirection.higher_is_better] = Field( + default=RankDirection.higher_is_better, + description="FusedMoment.score sorts descending; a larger score ranks better.", + ) + score_calibration: Literal[ScoreCalibration.ordering_only] = Field( + default=ScoreCalibration.ordering_only, + description=( + "The combined FusedMoment.score is ordering_only: valid for sorting " + "moments within one response, never a probability or confidence." + ), + ) class FusedMoment(ApplicationModel): moment_id: Sha256 | None = None - rank: int = Field(gt=0) - score: float = Field(gt=0) + rank: int = Field( + gt=0, + description="1-based combined rank across all searched channels; rank 1 is best.", + ) + score: float = Field( + gt=0, + description=( + "Combined reciprocal-rank-fusion score; larger ranks better. " + "Ordering-only, not a probability; see fusion.score_calibration." + ), + ) media_id: MediaId start: float = Field(ge=0) end: float = Field(gt=0) - modalities: tuple[Identifier, ...] + modalities: tuple[Identifier, ...] = Field( + description="Channels that contributed a hit to this moment.", + ) hits: tuple[SearchHit, ...] = Field(min_length=1) @model_validator(mode="after") @@ -993,6 +1107,14 @@ class FusedSearchResult(ApplicationModel): query_id: str = Field(min_length=1) query: SearchQuery modalities: tuple[Identifier, ...] + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description=( + "Meaning of the per-hit rank, score, and raw_distance carried by " + "each moment's hits. The combined moment score is described by " + "fusion.score_calibration and fusion.score_direction." + ), + ) moments: tuple[FusedMoment, ...] = () fusion: FusionProvenance evidence_delivery: "EvidenceDeliveryResult | None" = None @@ -1177,7 +1299,13 @@ class EvidenceDeliveryItem(ApplicationModel): media_id: MediaId generation_id: IndexGenerationId modalities: tuple[Identifier, ...] = Field(min_length=1) - score: float | None = None + score: float | None = Field( + default=None, + description=( + "Combined ordering-only fusion score copied from the source moment; " + "larger ranks better, not a probability." + ), + ) provenance: dict[str, JsonValue] = Field(default_factory=dict) state: EvidenceDeliveryState range: EvidenceRangeResolution | None = None diff --git a/src/vidxp/capabilities/search.py b/src/vidxp/capabilities/search.py index 9c57ddc..8505d71 100644 --- a/src/vidxp/capabilities/search.py +++ b/src/vidxp/capabilities/search.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Mapping +from vidxp.application_models import RetrievalScoring from vidxp.capabilities.schemas import SearchHit, SearchResult from vidxp.core.contracts import ( IndexConfig, @@ -143,6 +144,7 @@ def search_embeddings( query_id=query_id or stable_query_id(query, modality, config), query=query, modality=modality, + scoring=RetrievalScoring(distance_metric=config.vector_distance), hits=_to_hits(modality, rows, required_metadata), ) diff --git a/src/vidxp/cli_support.py b/src/vidxp/cli_support.py index 934c3b5..eadf8f2 100644 --- a/src/vidxp/cli_support.py +++ b/src/vidxp/cli_support.py @@ -147,11 +147,18 @@ def emit_search( if not result.moments: typer.echo("No matching moments found.") return - table = Table(title="Fused search results") + table = Table( + title="Fused search results", + caption=( + "Rank 1 = best match. Score = reciprocal-rank fusion " + "(higher ranks better); ordering-only, not a probability. " + f"Distance metric: {result.scoring.distance_metric}." + ), + ) table.add_column("Rank", justify="right") table.add_column("Start", justify="right") table.add_column("End", justify="right") - table.add_column("Score", justify="right") + table.add_column("Score (RRF)", justify="right") table.add_column("Video") table.add_column("Modalities") for moment in result.moments: diff --git a/src/vidxp/search_fusion.py b/src/vidxp/search_fusion.py index b7e4184..058f74b 100644 --- a/src/vidxp/search_fusion.py +++ b/src/vidxp/search_fusion.py @@ -7,6 +7,7 @@ FusedMoment, FusedSearchResult, FusionProvenance, + RetrievalScoring, SearchHit, SearchResult, ) @@ -190,6 +191,9 @@ def fuse_search_results( ), query=query, modalities=searched_modalities, + scoring=( + ordered_results[0].scoring if ordered_results else RetrievalScoring() + ), moments=moments, fusion=FusionProvenance( requested_modalities=requested_modalities, diff --git a/tests/test_search.py b/tests/test_search.py index accb7d8..6b43595 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -112,6 +112,35 @@ def test_score_is_strictly_monotonic_and_not_a_probability(self): self.assertGreater(distance_to_score(0.1), distance_to_score(0.2)) self.assertEqual(distance_to_score(2.5), -2.5) + def test_result_describes_its_metric_and_ordering_only_scores(self): + config = IndexConfig( + dataset="sample", + split="test", + run_id="run-1", + enabled_modalities=("speech",), + vector_distance="cosine", + ) + storage = FakeStorage([dialogue_row("run:video-1:dialogue:a", 0.1)]) + with patch( + "vidxp.capabilities.speech.operations.speech_embedding", + return_value=[0.5, 0.25], + ): + result = search_speech( + "fresh bread", + config=config, + runtime=self.runtime, + top_k=1, + storage=storage, + ) + + scoring = result.scoring + self.assertEqual(scoring.distance_metric, "cosine") + self.assertEqual(scoring.raw_distance_direction, "lower_is_better") + self.assertEqual(scoring.score_transform, "negated_distance") + self.assertEqual(scoring.score_direction, "higher_is_better") + self.assertEqual(scoring.score_calibration, "ordering_only") + self.assertEqual(scoring.hit_rank_direction, "lower_is_better") + def test_dialogue_query_uses_model_owned_query_prompt(self): encoder = Mock() encoder.encode_query.return_value = np.asarray([[0.5, 0.25]]) diff --git a/tests/test_search_fusion.py b/tests/test_search_fusion.py index 622e021..2f603a6 100644 --- a/tests/test_search_fusion.py +++ b/tests/test_search_fusion.py @@ -1,6 +1,11 @@ import unittest -from vidxp.application_models import SearchHit, SearchResult +from vidxp.application_models import ( + FusedSearchResult, + RetrievalScoring, + SearchHit, + SearchResult, +) from vidxp.search_fusion import RRF_RANK_CONSTANT, fuse_search_results @@ -109,6 +114,66 @@ def test_rewritten_atomic_query_identity_changes_fused_identity(self): self.assertNotEqual(first.query_id, second.query_id) + def test_fused_result_marks_every_score_ordering_only(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(result.scoring.score_calibration, "ordering_only") + self.assertEqual(result.fusion.score_calibration, "ordering_only") + self.assertEqual(result.fusion.score_direction, "higher_is_better") + + def test_fused_result_inherits_the_channel_distance_metric(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + scoring=RetrievalScoring(distance_metric="cosine"), + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(result.scoring.distance_metric, "cosine") + + def test_legacy_stored_result_without_scoring_still_loads(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + # Simulate a job result stored before the scoring descriptor existed. + payload = result.model_dump(mode="json") + payload.pop("scoring") + payload["fusion"].pop("score_direction") + payload["fusion"].pop("score_calibration") + + restored = FusedSearchResult.model_validate(payload) + + self.assertEqual(restored.scoring.distance_metric, "l2") + self.assertEqual(restored.scoring.score_calibration, "ordering_only") + self.assertEqual(restored.fusion.score_calibration, "ordering_only") + if __name__ == "__main__": unittest.main()