From f3fba922ba584cf116148a0982a8c9a6c286790e Mon Sep 17 00:00:00 2001 From: AhmedIrfan7 Date: Sat, 5 Sep 2026 00:35:56 +0500 Subject: [PATCH] feat(search): make retrieval and fusion scores self-describing Search responses returned several ranking numbers (a raw vector-store distance, a derived per-channel score, a per-channel rank, and a combined fusion score) with no statement of what each meant or which direction ranked better, and the derived scores could be mistaken for calibrated confidences. Add a RetrievalScoring descriptor, carried on both SearchResult and FusedSearchResult, that states the distance metric, the low/high ranking direction of raw_distance and score, the distance->score transform (negated_distance), and that both values are ordering_only rather than a probability. Extend FusionProvenance to declare the combined moment score as ordering_only and higher-is-better, and document requested vs searched channels. Add field descriptions to every ranking value on SearchHit, FusedMoment, and the evidence artifacts so the meaning is identical across the CLI, HTTP, MCP, stored job results, and evidence delivery. The CLI search table now labels the score column and captions the metric and ordering-only meaning. The change is additive: every new field is defaulted, so previously stored job results and existing FusedSearchResult payloads still validate. Scores remain uncalibrated; calibrated scoring is deferred to the end-to-end ranking evaluation in #76. Closes #90 --- docs/architecture/platform.md | 24 ++++- src/vidxp/application_models.py | 148 ++++++++++++++++++++++++++++--- src/vidxp/capabilities/search.py | 2 + src/vidxp/cli_support.py | 11 ++- src/vidxp/search_fusion.py | 4 + tests/test_search.py | 29 ++++++ tests/test_search_fusion.py | 67 +++++++++++++- 7 files changed, 271 insertions(+), 14 deletions(-) diff --git a/docs/architecture/platform.md b/docs/architecture/platform.md index a0b03b29..7b73c565 100644 --- a/docs/architecture/platform.md +++ b/docs/architecture/platform.md @@ -818,13 +818,35 @@ Results are normalized into `SearchHit`: - source/index generation - modality - start/end time -- score and score semantics +- `rank`, `score`, and `raw_distance` (see ranking values below) - displayable text/metadata - optional preview reference The application owns limits, score normalization, multimodal fusion and deterministic ordering. UI/API/MCP may request stricter limits but cannot loosen policy. +### Ranking values + +Every search response is self-describing about its numbers so callers never have to +guess which value they received or which direction ranks better. A `RetrievalScoring` +descriptor is attached to both `SearchResult` and `FusedSearchResult`, and the same +descriptor is returned identically across the CLI, HTTP, MCP, stored job results, and +evidence artifacts. + +- `raw_distance` is the raw vector-store distance under `scoring.distance_metric` + (`l2`, `cosine`, or `ip`); smaller is closer. +- `score` on a hit is derived as `score = -raw_distance` (`score_transform`), so larger + ranks better. +- A `SearchHit.rank` is 1-based within one modality channel; ranks from different + channels are not comparable. +- A `FusedMoment.rank`/`score` is the combined position and reciprocal-rank-fusion score + across channels; `fusion.searched_modalities` lists the channels that ran and + `FusedMoment.modalities` lists the channels that contributed to that moment. +- All of these are `ordering_only` (`scoring.score_calibration` and + `fusion.score_calibration`): valid for sorting within a single response, never a + probability or confidence. Calibrated scores are deferred to the end-to-end ranking + evaluation in issue #76. + Actor cluster and detection queries use the same pagination conventions. ## 16. Natural-language query layer diff --git a/src/vidxp/application_models.py b/src/vidxp/application_models.py index 395bde42..cffe6a53 100644 --- a/src/vidxp/application_models.py +++ b/src/vidxp/application_models.py @@ -809,6 +809,61 @@ class FusionProfile(StrEnum): reciprocal_rank = "rrf_v1" +class RankDirection(StrEnum): + """Whether smaller or larger values of a ranking quantity rank better.""" + + lower_is_better = "lower_is_better" + higher_is_better = "higher_is_better" + + +class ScoreCalibration(StrEnum): + """How much meaning a numeric score carries beyond ordering.""" + + ordering_only = "ordering_only" + + +class RetrievalScoring(ApplicationModel): + """Self-describing meaning of the ranking values in a search response. + + Callers see several numbers on a hit (a raw vector-store distance and a + derived score) and a rank. This descriptor states, in the payload itself, + which distance metric produced ``raw_distance``, which direction ranks + better, how ``score`` is derived from the distance, and that neither value + is calibrated. It is returned identically across the CLI, HTTP, MCP, + stored job results, and evidence artifacts so the meaning never drifts + between surfaces. Calibrated, probability-like scores are deferred to the + end-to-end ranking evaluation tracked in issue #76. + """ + + distance_metric: Literal["l2", "cosine", "ip"] = Field( + default="l2", + description="Vector-store distance space that produced each raw_distance.", + ) + raw_distance_direction: Literal[RankDirection.lower_is_better] = Field( + default=RankDirection.lower_is_better, + description="raw_distance sorts ascending; a smaller distance is closer.", + ) + score_transform: Literal["negated_distance"] = Field( + default="negated_distance", + description="Each hit score is derived as score = -raw_distance.", + ) + score_direction: Literal[RankDirection.higher_is_better] = Field( + default=RankDirection.higher_is_better, + description="score sorts descending; a larger score ranks better.", + ) + score_calibration: Literal[ScoreCalibration.ordering_only] = Field( + default=ScoreCalibration.ordering_only, + description=( + "raw_distance and score are ordering_only: valid for sorting hits " + "within one response, never a probability or confidence." + ), + ) + hit_rank_direction: Literal[RankDirection.lower_is_better] = Field( + default=RankDirection.lower_is_better, + description="A hit's per-channel rank starts at 1; rank 1 is the closest.", + ) + + class EvidenceDeliveryMode(StrEnum): none = "none" keyframes = "keyframes" @@ -840,7 +895,13 @@ class EvidenceBoardCandidate(ApplicationModel): representative_timestamp: float = Field(ge=0) frame_index: int | None = Field(default=None, ge=0) frame_match: "EvidenceFrameMatch" - score: float | None = None + score: float | None = Field( + default=None, + description=( + "Combined ordering-only fusion score copied from the source moment; " + "larger ranks better, not a probability." + ), + ) display_text: str | None = Field(default=None, max_length=512) provenance: dict[str, JsonValue] = Field(default_factory=dict) @@ -895,14 +956,32 @@ def _unique_modalities( class SearchHit(ApplicationModel): - rank: int = Field(gt=0) + rank: int = Field( + gt=0, + description=( + "1-based rank of this hit within its own modality channel " + "(rank 1 is closest). Ranks from different channels are not " + "comparable; the combined position is FusedMoment.rank." + ), + ) media_id: MediaId video_id: VideoId generation_id: IndexGenerationId start: float = Field(ge=0) end: float = Field(gt=0) - score: float - raw_distance: float + score: float = Field( + description=( + "Ordering-only channel score, score = -raw_distance, so larger " + "ranks better. Not a probability or confidence; see the response " + "scoring descriptor." + ), + ) + raw_distance: float = Field( + description=( + "Raw vector-store distance under the configured metric; smaller is " + "closer. See scoring.distance_metric for the metric." + ), + ) modality: str = Field(min_length=1) source_id: str = Field(min_length=1) metadata: dict[str, JsonValue] = Field(default_factory=dict) @@ -950,6 +1029,10 @@ class SearchResult(ApplicationModel): query_id: str = Field(min_length=1) query: str = Field(min_length=1) modality: str = Field(min_length=1) + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description="Meaning of the rank, score, and raw_distance on each hit.", + ) hits: tuple[SearchHit, ...] = () def to_dict(self) -> dict[str, Any]: @@ -963,18 +1046,49 @@ class FusionProvenance(ApplicationModel): profile: Literal[FusionProfile.reciprocal_rank] = FusionProfile.reciprocal_rank rank_constant: int = Field(default=60, gt=0) overlap_rule: Literal["connected_intervals"] = "connected_intervals" - requested_modalities: tuple[Identifier, ...] = () - searched_modalities: tuple[Identifier, ...] = () + requested_modalities: tuple[Identifier, ...] = Field( + default=(), + description="Channels the caller asked to search.", + ) + searched_modalities: tuple[Identifier, ...] = Field( + default=(), + description=( + "Channels actually run for this response. A moment's contributing " + "channels are the subset in FusedMoment.modalities." + ), + ) + score_direction: Literal[RankDirection.higher_is_better] = Field( + default=RankDirection.higher_is_better, + description="FusedMoment.score sorts descending; a larger score ranks better.", + ) + score_calibration: Literal[ScoreCalibration.ordering_only] = Field( + default=ScoreCalibration.ordering_only, + description=( + "The combined FusedMoment.score is ordering_only: valid for sorting " + "moments within one response, never a probability or confidence." + ), + ) class FusedMoment(ApplicationModel): moment_id: Sha256 | None = None - rank: int = Field(gt=0) - score: float = Field(gt=0) + rank: int = Field( + gt=0, + description="1-based combined rank across all searched channels; rank 1 is best.", + ) + score: float = Field( + gt=0, + description=( + "Combined reciprocal-rank-fusion score; larger ranks better. " + "Ordering-only, not a probability; see fusion.score_calibration." + ), + ) media_id: MediaId start: float = Field(ge=0) end: float = Field(gt=0) - modalities: tuple[Identifier, ...] + modalities: tuple[Identifier, ...] = Field( + description="Channels that contributed a hit to this moment.", + ) hits: tuple[SearchHit, ...] = Field(min_length=1) @model_validator(mode="after") @@ -993,6 +1107,14 @@ class FusedSearchResult(ApplicationModel): query_id: str = Field(min_length=1) query: SearchQuery modalities: tuple[Identifier, ...] + scoring: RetrievalScoring = Field( + default_factory=RetrievalScoring, + description=( + "Meaning of the per-hit rank, score, and raw_distance carried by " + "each moment's hits. The combined moment score is described by " + "fusion.score_calibration and fusion.score_direction." + ), + ) moments: tuple[FusedMoment, ...] = () fusion: FusionProvenance evidence_delivery: "EvidenceDeliveryResult | None" = None @@ -1177,7 +1299,13 @@ class EvidenceDeliveryItem(ApplicationModel): media_id: MediaId generation_id: IndexGenerationId modalities: tuple[Identifier, ...] = Field(min_length=1) - score: float | None = None + score: float | None = Field( + default=None, + description=( + "Combined ordering-only fusion score copied from the source moment; " + "larger ranks better, not a probability." + ), + ) provenance: dict[str, JsonValue] = Field(default_factory=dict) state: EvidenceDeliveryState range: EvidenceRangeResolution | None = None diff --git a/src/vidxp/capabilities/search.py b/src/vidxp/capabilities/search.py index 9c57ddc3..8505d710 100644 --- a/src/vidxp/capabilities/search.py +++ b/src/vidxp/capabilities/search.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Mapping +from vidxp.application_models import RetrievalScoring from vidxp.capabilities.schemas import SearchHit, SearchResult from vidxp.core.contracts import ( IndexConfig, @@ -143,6 +144,7 @@ def search_embeddings( query_id=query_id or stable_query_id(query, modality, config), query=query, modality=modality, + scoring=RetrievalScoring(distance_metric=config.vector_distance), hits=_to_hits(modality, rows, required_metadata), ) diff --git a/src/vidxp/cli_support.py b/src/vidxp/cli_support.py index 934c3b5d..eadf8f2c 100644 --- a/src/vidxp/cli_support.py +++ b/src/vidxp/cli_support.py @@ -147,11 +147,18 @@ def emit_search( if not result.moments: typer.echo("No matching moments found.") return - table = Table(title="Fused search results") + table = Table( + title="Fused search results", + caption=( + "Rank 1 = best match. Score = reciprocal-rank fusion " + "(higher ranks better); ordering-only, not a probability. " + f"Distance metric: {result.scoring.distance_metric}." + ), + ) table.add_column("Rank", justify="right") table.add_column("Start", justify="right") table.add_column("End", justify="right") - table.add_column("Score", justify="right") + table.add_column("Score (RRF)", justify="right") table.add_column("Video") table.add_column("Modalities") for moment in result.moments: diff --git a/src/vidxp/search_fusion.py b/src/vidxp/search_fusion.py index b7e41849..058f74b5 100644 --- a/src/vidxp/search_fusion.py +++ b/src/vidxp/search_fusion.py @@ -7,6 +7,7 @@ FusedMoment, FusedSearchResult, FusionProvenance, + RetrievalScoring, SearchHit, SearchResult, ) @@ -190,6 +191,9 @@ def fuse_search_results( ), query=query, modalities=searched_modalities, + scoring=( + ordered_results[0].scoring if ordered_results else RetrievalScoring() + ), moments=moments, fusion=FusionProvenance( requested_modalities=requested_modalities, diff --git a/tests/test_search.py b/tests/test_search.py index accb7d84..6b43595f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -112,6 +112,35 @@ def test_score_is_strictly_monotonic_and_not_a_probability(self): self.assertGreater(distance_to_score(0.1), distance_to_score(0.2)) self.assertEqual(distance_to_score(2.5), -2.5) + def test_result_describes_its_metric_and_ordering_only_scores(self): + config = IndexConfig( + dataset="sample", + split="test", + run_id="run-1", + enabled_modalities=("speech",), + vector_distance="cosine", + ) + storage = FakeStorage([dialogue_row("run:video-1:dialogue:a", 0.1)]) + with patch( + "vidxp.capabilities.speech.operations.speech_embedding", + return_value=[0.5, 0.25], + ): + result = search_speech( + "fresh bread", + config=config, + runtime=self.runtime, + top_k=1, + storage=storage, + ) + + scoring = result.scoring + self.assertEqual(scoring.distance_metric, "cosine") + self.assertEqual(scoring.raw_distance_direction, "lower_is_better") + self.assertEqual(scoring.score_transform, "negated_distance") + self.assertEqual(scoring.score_direction, "higher_is_better") + self.assertEqual(scoring.score_calibration, "ordering_only") + self.assertEqual(scoring.hit_rank_direction, "lower_is_better") + def test_dialogue_query_uses_model_owned_query_prompt(self): encoder = Mock() encoder.encode_query.return_value = np.asarray([[0.5, 0.25]]) diff --git a/tests/test_search_fusion.py b/tests/test_search_fusion.py index 622e0210..2f603a6e 100644 --- a/tests/test_search_fusion.py +++ b/tests/test_search_fusion.py @@ -1,6 +1,11 @@ import unittest -from vidxp.application_models import SearchHit, SearchResult +from vidxp.application_models import ( + FusedSearchResult, + RetrievalScoring, + SearchHit, + SearchResult, +) from vidxp.search_fusion import RRF_RANK_CONSTANT, fuse_search_results @@ -109,6 +114,66 @@ def test_rewritten_atomic_query_identity_changes_fused_identity(self): self.assertNotEqual(first.query_id, second.query_id) + def test_fused_result_marks_every_score_ordering_only(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(result.scoring.score_calibration, "ordering_only") + self.assertEqual(result.fusion.score_calibration, "ordering_only") + self.assertEqual(result.fusion.score_direction, "higher_is_better") + + def test_fused_result_inherits_the_channel_distance_metric(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + scoring=RetrievalScoring(distance_metric="cosine"), + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(result.scoring.distance_metric, "cosine") + + def test_legacy_stored_result_without_scoring_still_loads(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1, 2, "scene:1"),), + ) + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + # Simulate a job result stored before the scoring descriptor existed. + payload = result.model_dump(mode="json") + payload.pop("scoring") + payload["fusion"].pop("score_direction") + payload["fusion"].pop("score_calibration") + + restored = FusedSearchResult.model_validate(payload) + + self.assertEqual(restored.scoring.distance_metric, "l2") + self.assertEqual(restored.scoring.score_calibration, "ordering_only") + self.assertEqual(restored.fusion.score_calibration, "ordering_only") + if __name__ == "__main__": unittest.main()