From a8e1c8bd00df6a027ed53936b18dc8ffb1f15183 Mon Sep 17 00:00:00 2001 From: Edgar Babajanyan Date: Wed, 2 Sep 2026 12:10:26 -0700 Subject: [PATCH] Add captain-text pipeline results (Captain file-search API) --- .../captain_text_pipeline.py | 113 ++++++++++++++++++ .../vidore_v3_computer_science.json | 70 +++++++++++ .../captain-text/vidore_v3_energy.json | 70 +++++++++++ .../captain-text/vidore_v3_finance_en.json | 70 +++++++++++ .../captain-text/vidore_v3_finance_fr.json | 70 +++++++++++ .../metrics/captain-text/vidore_v3_hr.json | 70 +++++++++++ .../captain-text/vidore_v3_industrial.json | 70 +++++++++++ .../vidore_v3_pharmaceuticals.json | 70 +++++++++++ .../captain-text/vidore_v3_physics.json | 70 +++++++++++ .../captain-text/description.json | 54 +++++++++ 10 files changed, 727 insertions(+) create mode 100644 pipeline_implementations/captain_text_pipeline.py create mode 100644 results/metrics/captain-text/vidore_v3_computer_science.json create mode 100644 results/metrics/captain-text/vidore_v3_energy.json create mode 100644 results/metrics/captain-text/vidore_v3_finance_en.json create mode 100644 results/metrics/captain-text/vidore_v3_finance_fr.json create mode 100644 results/metrics/captain-text/vidore_v3_hr.json create mode 100644 results/metrics/captain-text/vidore_v3_industrial.json create mode 100644 results/metrics/captain-text/vidore_v3_pharmaceuticals.json create mode 100644 results/metrics/captain-text/vidore_v3_physics.json create mode 100644 results/pipeline_descriptions/captain-text/description.json diff --git a/pipeline_implementations/captain_text_pipeline.py b/pipeline_implementations/captain_text_pipeline.py new file mode 100644 index 00000000..b8e48d0a --- /dev/null +++ b/pipeline_implementations/captain_text_pipeline.py @@ -0,0 +1,113 @@ +"""Captain (runcaptain.com) managed file-search API — text pipeline. + +Ingestion: each ViDoRe v3 corpus page's provided NV-Ingest markdown is indexed +as one document named `{corpus_id}.md`, prefixed with a one-line +`# {doc_id} - page {n}` header so chunks carry document identity. Captain then +does its own chunking, LLM enrichment, and embedding server-side. + +Retrieval: hybrid BM25 + dense retrieval with voyage-rerank-2.5 reranking +(candidate pool 200), one API call per query. Pages are ranked by their best +chunk score (MaxP); returned document filenames map back to corpus ids. + +Requires the CAPTAIN_API_KEY environment variable (closed API; free keys at +https://www.captain.dev). See description.json for run/timing details. +""" +import json +import os +import time +from concurrent.futures import ThreadPoolExecutor + +import requests + +from vidore_benchmark.pipeline_evaluation.base_pipeline import BasePipeline + +API = os.environ.get("CAPTAIN_API", "https://api.runcaptain.com") +KEY = os.environ["CAPTAIN_API_KEY"] +EPS = 1e-6 + + +class CaptainTextPipeline(BasePipeline): + def __init__(self, top_k: int = 100, workers: int = 10, **kwargs): + self.top_k = top_k + self.workers = workers + self.s = requests.Session() + self.s.headers.update({"Authorization": f"Bearer {KEY}", + "Content-Type": "application/json"}) + self.collection = None + + # ------------------------------------------------------------------ index + def index(self, corpus_ids, corpus_images, corpus_texts, dataset_name=None): + ds = (dataset_name or "vidore_v3").rsplit("vidore_v3_", 1)[-1] + self.collection = f"vidore-{ds}-text" + r = self.s.put(f"{API}/v2/collections/{self.collection}", json={}, timeout=60) + if r.status_code == 409 or (r.status_code == 200 and self._doc_count() >= len(corpus_ids)): + return # already indexed (pre-built corpus reused across runs) + if r.status_code not in (200, 201): + r.raise_for_status() + + def _put(pair): + cid, text = pair + body = {"content": text or " ", "file_name": f"{cid}.md"} + for attempt in range(3): + resp = self.s.post(f"{API}/v2/collections/{self.collection}/index/text", + json=body, timeout=120) + if resp.status_code in (200, 201, 202): + return + time.sleep(2 * (attempt + 1)) + resp.raise_for_status() + + with ThreadPoolExecutor(max_workers=self.workers) as ex: + list(ex.map(_put, zip(corpus_ids, corpus_texts))) + # poll until documents are queryable + deadline = time.time() + 3600 + while time.time() < deadline and self._doc_count() < len(corpus_ids): + time.sleep(15) + + def _doc_count(self): + try: + r = self.s.get(f"{API}/v2/collections/{self.collection}/documents?limit=1", + timeout=60) + return (r.json().get("pagination") or {}).get("total") or \ + len(r.json().get("documents") or []) + except Exception: + return 0 + + # ----------------------------------------------------------------- search + def _one(self, qid, query): + body = {"query": query, "limit": 100, + "rerank": {"enabled": True, "candidate_limit": 200}, + "include": {"document": True, "metadata": False}} + if self.collection.endswith("finance_fr-text"): + # a-priori rule for the cross-lingual set (English queries, French + # corpus): keyword retrieval contributes nothing, use dense only. + body["semantic_ratio"] = 1.0 + for attempt in range(3): + try: + r = self.s.post(f"{API}/v3/collections/{self.collection}/query", + json=body, timeout=120) + r.raise_for_status() + break + except Exception: + if attempt == 2: + raise + time.sleep(3 * (attempt + 1)) + d = r.json() + assert not (d.get("rerank") or {}).get("fallback"), "rerank fallback — rerun" + agg = {} + for res in d.get("results", []): + fname = ((res.get("document") or {}).get("filename") or "").rsplit("/", 1)[-1] + stem = fname[:-3] if fname.endswith(".md") else fname + if not stem.isdigit(): + continue + score = float(res.get("score") or 0.0) + agg[stem] = max(agg.get(stem, 0.0), score) # MaxP page aggregation + ranked = sorted(agg.items(), key=lambda kv: -kv[1])[: self.top_k] + return qid, {cid: s - i * EPS for i, (cid, s) in enumerate(ranked)} + + def search(self, query_ids, queries): + out = {} + t0 = time.time() + with ThreadPoolExecutor(max_workers=self.workers) as ex: + for qid, run in ex.map(lambda a: self._one(*a), zip(query_ids, queries)): + out[str(qid)] = run + return out, {"total_retrieval_time_seconds": round(time.time() - t0, 2)} diff --git a/results/metrics/captain-text/vidore_v3_computer_science.json b/results/metrics/captain-text/vidore_v3_computer_science.json new file mode 100644 index 00000000..8b8d6b17 --- /dev/null +++ b/results/metrics/captain-text/vidore_v3_computer_science.json @@ -0,0 +1,70 @@ +{ + "dataset": "vidore/vidore_v3_computer_science", + "split": "test", + "language": "english", + "pipeline_type": null, + "module_path": "captain_official.py", + "class_name": "CaptainTextOfficial", + "pipeline_args": {}, + "aggregated_metrics": { + "overall": { + "map": 0.7895771479970486, + "recip_rank": 0.9343466223698782, + "P_1": 0.8930232558139535, + "P_5": 0.5469767441860466, + "P_10": 0.35953488372093023, + "P_20": 0.2111627906976744, + "recall_1": 0.3411533815022187, + "recall_5": 0.7229698854117459, + "recall_10": 0.8468431697501465, + "recall_20": 0.9193194789706417, + "recall_50": 0.9434451723986608, + "recall_100": 0.9521790225278597, + "ndcg_cut_1": 0.8348837209302326, + "ndcg_cut_5": 0.8248622688551305, + "ndcg_cut_10": 0.8411160239742446, + "ndcg_cut_20": 0.8648934625660156, + "ndcg_cut_100": 0.8761385960959897, + "map_cut_1": 0.3411533815022187, + "map_cut_10": 0.7474295024271491, + "map_cut_100": 0.7895771479970486 + }, + "by_language": { + "english": { + "map": 0.7895771479970486, + "recip_rank": 0.9343466223698782, + "P_1": 0.8930232558139535, + "P_5": 0.5469767441860466, + "P_10": 0.35953488372093023, + "P_20": 0.2111627906976744, + "recall_1": 0.3411533815022187, + "recall_5": 0.7229698854117459, + "recall_10": 0.8468431697501465, + "recall_20": 0.9193194789706417, + "recall_50": 0.9434451723986608, + "recall_100": 0.9521790225278597, + "ndcg_cut_1": 0.8348837209302326, + "ndcg_cut_5": 0.8248622688551305, + "ndcg_cut_10": 0.8411160239742446, + "ndcg_cut_20": 0.8648934625660156, + "ndcg_cut_100": 0.8761385960959897, + "map_cut_1": 0.3411533815022187, + "map_cut_10": 0.7474295024271491, + "map_cut_100": 0.7895771479970486, + "num_queries": 215 + } + }, + "timing": { + "total_retrieval_time_milliseconds": 102711.82107925415, + "indexing_time_milliseconds": 0.0, + "search_time_milliseconds": 102711.82107925415, + "num_queries": 215, + "num_corpus": 1360, + "indexing_throughput_ms_per_doc": 0.0, + "search_throughput_ms_per_query": 477.72940036862394 + }, + "infos": { + "total_retrieval_time_seconds": 102.71 + } + } +} \ No newline at end of file diff --git a/results/metrics/captain-text/vidore_v3_energy.json b/results/metrics/captain-text/vidore_v3_energy.json new file mode 100644 index 00000000..051bddde --- /dev/null +++ b/results/metrics/captain-text/vidore_v3_energy.json @@ -0,0 +1,70 @@ +{ + "dataset": "vidore/vidore_v3_energy", + "split": "test", + "language": "english", + "pipeline_type": null, + "module_path": "captain_official.py", + "class_name": "CaptainTextOfficial", + "pipeline_args": {}, + "aggregated_metrics": { + "overall": { + "map": 0.6536825354885418, + "recip_rank": 0.7953532184948794, + "P_1": 0.7045454545454546, + "P_5": 0.30714285714285716, + "P_10": 0.20194805194805196, + "P_20": 0.12224025974025973, + "recall_1": 0.4144982449265048, + "recall_5": 0.6901633096337426, + "recall_10": 0.7840659496211037, + "recall_20": 0.8482336284089727, + "recall_50": 0.8927673748322279, + "recall_100": 0.8936360060375704, + "ndcg_cut_1": 0.672077922077922, + "ndcg_cut_5": 0.6981245103225042, + "ndcg_cut_10": 0.7256963666681007, + "ndcg_cut_20": 0.7443411803975708, + "ndcg_cut_100": 0.7580956424452949, + "map_cut_1": 0.4144982449265048, + "map_cut_10": 0.6276145516677658, + "map_cut_100": 0.6536825354885418 + }, + "by_language": { + "english": { + "map": 0.6536825354885418, + "recip_rank": 0.7953532184948794, + "P_1": 0.7045454545454546, + "P_5": 0.30714285714285716, + "P_10": 0.20194805194805196, + "P_20": 0.12224025974025973, + "recall_1": 0.4144982449265048, + "recall_5": 0.6901633096337426, + "recall_10": 0.7840659496211037, + "recall_20": 0.8482336284089727, + "recall_50": 0.8927673748322279, + "recall_100": 0.8936360060375704, + "ndcg_cut_1": 0.672077922077922, + "ndcg_cut_5": 0.6981245103225042, + "ndcg_cut_10": 0.7256963666681007, + "ndcg_cut_20": 0.7443411803975708, + "ndcg_cut_100": 0.7580956424452949, + "map_cut_1": 0.4144982449265048, + "map_cut_10": 0.6276145516677658, + "map_cut_100": 0.6536825354885418, + "num_queries": 308 + } + }, + "timing": { + "total_retrieval_time_milliseconds": 200609.8837852478, + "indexing_time_milliseconds": 0.0, + "search_time_milliseconds": 200609.8837852478, + "num_queries": 308, + "num_corpus": 2225, + "indexing_throughput_ms_per_doc": 0.0, + "search_throughput_ms_per_query": 651.3307915105448 + }, + "infos": { + "total_retrieval_time_seconds": 200.61 + } + } +} \ No newline at end of file diff --git a/results/metrics/captain-text/vidore_v3_finance_en.json b/results/metrics/captain-text/vidore_v3_finance_en.json new file mode 100644 index 00000000..a41800f3 --- /dev/null +++ b/results/metrics/captain-text/vidore_v3_finance_en.json @@ -0,0 +1,70 @@ +{ + "dataset": "vidore/vidore_v3_finance_en", + "split": "test", + "language": "english", + "pipeline_type": null, + "module_path": "captain_official.py", + "class_name": "CaptainTextOfficial", + "pipeline_args": {}, + "aggregated_metrics": { + "overall": { + "map": 0.6202737162091443, + "recip_rank": 0.8237355127135183, + "P_1": 0.7249190938511327, + "P_5": 0.39158576051779936, + "P_10": 0.254368932038835, + "P_20": 0.15436893203883495, + "recall_1": 0.3263296696368024, + "recall_5": 0.6268750240355939, + "recall_10": 0.7321575400123749, + "recall_20": 0.8142189300947454, + "recall_50": 0.8884690862789092, + "recall_100": 0.8987406269830598, + "ndcg_cut_1": 0.6844660194174758, + "ndcg_cut_5": 0.676064451859597, + "ndcg_cut_10": 0.6956430294972429, + "ndcg_cut_20": 0.7208042805404912, + "ndcg_cut_100": 0.7498745601698638, + "map_cut_1": 0.3263296696368024, + "map_cut_10": 0.582357619891026, + "map_cut_100": 0.6202737162091443 + }, + "by_language": { + "english": { + "map": 0.6202737162091443, + "recip_rank": 0.8237355127135183, + "P_1": 0.7249190938511327, + "P_5": 0.39158576051779936, + "P_10": 0.254368932038835, + "P_20": 0.15436893203883495, + "recall_1": 0.3263296696368024, + "recall_5": 0.6268750240355939, + "recall_10": 0.7321575400123749, + "recall_20": 0.8142189300947454, + "recall_50": 0.8884690862789092, + "recall_100": 0.8987406269830598, + "ndcg_cut_1": 0.6844660194174758, + "ndcg_cut_5": 0.676064451859597, + "ndcg_cut_10": 0.6956430294972429, + "ndcg_cut_20": 0.7208042805404912, + "ndcg_cut_100": 0.7498745601698638, + "map_cut_1": 0.3263296696368024, + "map_cut_10": 0.582357619891026, + "map_cut_100": 0.6202737162091443, + "num_queries": 309 + } + }, + "timing": { + "total_retrieval_time_milliseconds": 187896.84796333313, + "indexing_time_milliseconds": 0.0, + "search_time_milliseconds": 187896.84796333313, + "num_queries": 309, + "num_corpus": 2942, + "indexing_throughput_ms_per_doc": 0.0, + "search_throughput_ms_per_query": 608.0804141208192 + }, + "infos": { + "total_retrieval_time_seconds": 187.89 + } + } +} \ No newline at end of file diff --git a/results/metrics/captain-text/vidore_v3_finance_fr.json b/results/metrics/captain-text/vidore_v3_finance_fr.json new file mode 100644 index 00000000..f544a34f --- /dev/null +++ b/results/metrics/captain-text/vidore_v3_finance_fr.json @@ -0,0 +1,70 @@ +{ + "dataset": "vidore/vidore_v3_finance_fr", + "split": "test", + "language": "english", + "pipeline_type": null, + "module_path": "captain_official.py", + "class_name": "CaptainTextOfficial", + "pipeline_args": {}, + "aggregated_metrics": { + "overall": { + "map": 0.4669575910516232, + "recip_rank": 0.6099577855693166, + "P_1": 0.471875, + "P_5": 0.284375, + "P_10": 0.1984375, + "P_20": 0.1290625, + "recall_1": 0.23675217860340142, + "recall_5": 0.487116772782134, + "recall_10": 0.6107233436963896, + "recall_20": 0.7103499539741781, + "recall_50": 0.7881712284188012, + "recall_100": 0.7955077805668342, + "ndcg_cut_1": 0.4625, + "ndcg_cut_5": 0.5068798710194243, + "ndcg_cut_10": 0.5405518905269726, + "ndcg_cut_20": 0.5743090322256234, + "ndcg_cut_100": 0.6030776268687237, + "map_cut_1": 0.23675217860340142, + "map_cut_10": 0.4310399960052096, + "map_cut_100": 0.4669575910516232 + }, + "by_language": { + "english": { + "map": 0.4669575910516232, + "recip_rank": 0.6099577855693166, + "P_1": 0.471875, + "P_5": 0.284375, + "P_10": 0.1984375, + "P_20": 0.1290625, + "recall_1": 0.23675217860340142, + "recall_5": 0.487116772782134, + "recall_10": 0.6107233436963896, + "recall_20": 0.7103499539741781, + "recall_50": 0.7881712284188012, + "recall_100": 0.7955077805668342, + "ndcg_cut_1": 0.4625, + "ndcg_cut_5": 0.5068798710194243, + "ndcg_cut_10": 0.5405518905269726, + "ndcg_cut_20": 0.5743090322256234, + "ndcg_cut_100": 0.6030776268687237, + "map_cut_1": 0.23675217860340142, + "map_cut_10": 0.4310399960052096, + "map_cut_100": 0.4669575910516232, + "num_queries": 320 + } + }, + "timing": { + "total_retrieval_time_milliseconds": 361213.0901813507, + "indexing_time_milliseconds": 0.0, + "search_time_milliseconds": 361213.0901813507, + "num_queries": 320, + "num_corpus": 2384, + "indexing_throughput_ms_per_doc": 0.0, + "search_throughput_ms_per_query": 1128.790906816721 + }, + "infos": { + "total_retrieval_time_seconds": 361.21 + } + } +} \ No newline at end of file diff --git a/results/metrics/captain-text/vidore_v3_hr.json b/results/metrics/captain-text/vidore_v3_hr.json new file mode 100644 index 00000000..3e6eec2f --- /dev/null +++ b/results/metrics/captain-text/vidore_v3_hr.json @@ -0,0 +1,70 @@ +{ + "dataset": "vidore/vidore_v3_hr", + "split": "test", + "language": "english", + "pipeline_type": null, + "module_path": "captain_official.py", + "class_name": "CaptainTextOfficial", + "pipeline_args": {}, + "aggregated_metrics": { + "overall": { + "map": 0.6274233803881407, + "recip_rank": 0.8097326654251493, + "P_1": 0.7232704402515723, + "P_5": 0.39622641509433965, + "P_10": 0.28144654088050314, + "P_20": 0.18553459119496854, + "recall_1": 0.3079605818839273, + "recall_5": 0.5960341437039081, + "recall_10": 0.7207064856660748, + "recall_20": 0.8232929090209616, + "recall_50": 0.9028642813276655, + "recall_100": 0.9161683267509936, + "ndcg_cut_1": 0.6933962264150944, + "ndcg_cut_5": 0.6797913980742488, + "ndcg_cut_10": 0.7040739749079685, + "ndcg_cut_20": 0.7307441109079864, + "ndcg_cut_100": 0.7640769661114377, + "map_cut_1": 0.3079605818839273, + "map_cut_10": 0.5695327541576074, + "map_cut_100": 0.6274233803881407 + }, + "by_language": { + "english": { + "map": 0.6274233803881407, + "recip_rank": 0.8097326654251493, + "P_1": 0.7232704402515723, + "P_5": 0.39622641509433965, + "P_10": 0.28144654088050314, + "P_20": 0.18553459119496854, + "recall_1": 0.3079605818839273, + "recall_5": 0.5960341437039081, + "recall_10": 0.7207064856660748, + "recall_20": 0.8232929090209616, + "recall_50": 0.9028642813276655, + "recall_100": 0.9161683267509936, + "ndcg_cut_1": 0.6933962264150944, + "ndcg_cut_5": 0.6797913980742488, + "ndcg_cut_10": 0.7040739749079685, + "ndcg_cut_20": 0.7307441109079864, + "ndcg_cut_100": 0.7640769661114377, + "map_cut_1": 0.3079605818839273, + "map_cut_10": 0.5695327541576074, + "map_cut_100": 0.6274233803881407, + "num_queries": 318 + } + }, + "timing": { + "total_retrieval_time_milliseconds": 202252.2418498993, + "indexing_time_milliseconds": 0.0, + "search_time_milliseconds": 202252.2418498993, + "num_queries": 318, + "num_corpus": 1110, + "indexing_throughput_ms_per_doc": 0.0, + "search_throughput_ms_per_query": 636.0133391506267 + }, + "infos": { + "total_retrieval_time_seconds": 202.25 + } + } +} \ No newline at end of file diff --git a/results/metrics/captain-text/vidore_v3_industrial.json b/results/metrics/captain-text/vidore_v3_industrial.json new file mode 100644 index 00000000..c2faeb84 --- /dev/null +++ b/results/metrics/captain-text/vidore_v3_industrial.json @@ -0,0 +1,70 @@ +{ + "dataset": "vidore/vidore_v3_industrial", + "split": "test", + "language": "english", + "pipeline_type": null, + "module_path": "captain_official.py", + "class_name": "CaptainTextOfficial", + "pipeline_args": {}, + "aggregated_metrics": { + "overall": { + "map": 0.5271826741177748, + "recip_rank": 0.6959340621382915, + "P_1": 0.6042402826855123, + "P_5": 0.30459363957597174, + "P_10": 0.20388692579505302, + "P_20": 0.1314487632508834, + "recall_1": 0.32133334516934603, + "recall_5": 0.533691911212968, + "recall_10": 0.6170053240775453, + "recall_20": 0.6970519367686638, + "recall_50": 0.769864568822418, + "recall_100": 0.7985257236618177, + "ndcg_cut_1": 0.5936395759717314, + "ndcg_cut_5": 0.5870786224647008, + "ndcg_cut_10": 0.6001976979086904, + "ndcg_cut_20": 0.6237051956545665, + "ndcg_cut_100": 0.6596380107263818, + "map_cut_1": 0.32133334516934603, + "map_cut_10": 0.4907752155192156, + "map_cut_100": 0.5271826741177748 + }, + "by_language": { + "english": { + "map": 0.5271826741177748, + "recip_rank": 0.6959340621382915, + "P_1": 0.6042402826855123, + "P_5": 0.30459363957597174, + "P_10": 0.20388692579505302, + "P_20": 0.1314487632508834, + "recall_1": 0.32133334516934603, + "recall_5": 0.533691911212968, + "recall_10": 0.6170053240775453, + "recall_20": 0.6970519367686638, + "recall_50": 0.769864568822418, + "recall_100": 0.7985257236618177, + "ndcg_cut_1": 0.5936395759717314, + "ndcg_cut_5": 0.5870786224647008, + "ndcg_cut_10": 0.6001976979086904, + "ndcg_cut_20": 0.6237051956545665, + "ndcg_cut_100": 0.6596380107263818, + "map_cut_1": 0.32133334516934603, + "map_cut_10": 0.4907752155192156, + "map_cut_100": 0.5271826741177748, + "num_queries": 283 + } + }, + "timing": { + "total_retrieval_time_milliseconds": 156918.2529449463, + "indexing_time_milliseconds": 0.0, + "search_time_milliseconds": 156918.2529449463, + "num_queries": 283, + "num_corpus": 5244, + "indexing_throughput_ms_per_doc": 0.0, + "search_throughput_ms_per_query": 554.4814591694216 + }, + "infos": { + "total_retrieval_time_seconds": 156.92 + } + } +} \ No newline at end of file diff --git a/results/metrics/captain-text/vidore_v3_pharmaceuticals.json b/results/metrics/captain-text/vidore_v3_pharmaceuticals.json new file mode 100644 index 00000000..c61707ef --- /dev/null +++ b/results/metrics/captain-text/vidore_v3_pharmaceuticals.json @@ -0,0 +1,70 @@ +{ + "dataset": "vidore/vidore_v3_pharmaceuticals", + "split": "test", + "language": "english", + "pipeline_type": null, + "module_path": "captain_official.py", + "class_name": "CaptainTextOfficial", + "pipeline_args": {}, + "aggregated_metrics": { + "overall": { + "map": 0.6637846410444629, + "recip_rank": 0.8249300481164096, + "P_1": 0.7335164835164835, + "P_5": 0.37692307692307697, + "P_10": 0.2642857142857143, + "P_20": 0.16909340659340658, + "recall_1": 0.36593884546462446, + "recall_5": 0.6366862774048556, + "recall_10": 0.7531144579647144, + "recall_20": 0.8450256739331408, + "recall_50": 0.9225142905854605, + "recall_100": 0.9494620794591331, + "ndcg_cut_1": 0.7046703296703297, + "ndcg_cut_5": 0.7013520267446066, + "ndcg_cut_10": 0.7254693214451255, + "ndcg_cut_20": 0.7546277145032556, + "ndcg_cut_100": 0.7923694042717064, + "map_cut_1": 0.36593884546462446, + "map_cut_10": 0.6116334639623537, + "map_cut_100": 0.6637846410444629 + }, + "by_language": { + "english": { + "map": 0.6637846410444629, + "recip_rank": 0.8249300481164096, + "P_1": 0.7335164835164835, + "P_5": 0.37692307692307697, + "P_10": 0.2642857142857143, + "P_20": 0.16909340659340658, + "recall_1": 0.36593884546462446, + "recall_5": 0.6366862774048556, + "recall_10": 0.7531144579647144, + "recall_20": 0.8450256739331408, + "recall_50": 0.9225142905854605, + "recall_100": 0.9494620794591331, + "ndcg_cut_1": 0.7046703296703297, + "ndcg_cut_5": 0.7013520267446066, + "ndcg_cut_10": 0.7254693214451255, + "ndcg_cut_20": 0.7546277145032556, + "ndcg_cut_100": 0.7923694042717064, + "map_cut_1": 0.36593884546462446, + "map_cut_10": 0.6116334639623537, + "map_cut_100": 0.6637846410444629, + "num_queries": 364 + } + }, + "timing": { + "total_retrieval_time_milliseconds": 132626.886844635, + "indexing_time_milliseconds": 0.0, + "search_time_milliseconds": 132626.886844635, + "num_queries": 364, + "num_corpus": 2313, + "indexing_throughput_ms_per_doc": 0.0, + "search_throughput_ms_per_query": 364.35957924350276 + }, + "infos": { + "total_retrieval_time_seconds": 132.63 + } + } +} \ No newline at end of file diff --git a/results/metrics/captain-text/vidore_v3_physics.json b/results/metrics/captain-text/vidore_v3_physics.json new file mode 100644 index 00000000..1621290a --- /dev/null +++ b/results/metrics/captain-text/vidore_v3_physics.json @@ -0,0 +1,70 @@ +{ + "dataset": "vidore/vidore_v3_physics", + "split": "test", + "language": "english", + "pipeline_type": null, + "module_path": "captain_official.py", + "class_name": "CaptainTextOfficial", + "pipeline_args": {}, + "aggregated_metrics": { + "overall": { + "map": 0.42174268883392935, + "recip_rank": 0.6177577856865648, + "P_1": 0.4602649006622517, + "P_5": 0.3172185430463576, + "P_10": 0.2599337748344371, + "P_20": 0.1922185430463576, + "recall_1": 0.15246203676949172, + "recall_5": 0.38951482138996124, + "recall_10": 0.5159716740592408, + "recall_20": 0.6687558970168914, + "recall_50": 0.8021027300144477, + "recall_100": 0.8460271409508826, + "ndcg_cut_1": 0.43874172185430466, + "ndcg_cut_5": 0.46473798693869783, + "ndcg_cut_10": 0.49456741918679326, + "ndcg_cut_20": 0.5431442270597209, + "ndcg_cut_100": 0.6051506737903127, + "map_cut_1": 0.15246203676949172, + "map_cut_10": 0.3429299573876064, + "map_cut_100": 0.42174268883392935 + }, + "by_language": { + "english": { + "map": 0.42174268883392935, + "recip_rank": 0.6177577856865648, + "P_1": 0.4602649006622517, + "P_5": 0.3172185430463576, + "P_10": 0.2599337748344371, + "P_20": 0.1922185430463576, + "recall_1": 0.15246203676949172, + "recall_5": 0.38951482138996124, + "recall_10": 0.5159716740592408, + "recall_20": 0.6687558970168914, + "recall_50": 0.8021027300144477, + "recall_100": 0.8460271409508826, + "ndcg_cut_1": 0.43874172185430466, + "ndcg_cut_5": 0.46473798693869783, + "ndcg_cut_10": 0.49456741918679326, + "ndcg_cut_20": 0.5431442270597209, + "ndcg_cut_100": 0.6051506737903127, + "map_cut_1": 0.15246203676949172, + "map_cut_10": 0.3429299573876064, + "map_cut_100": 0.42174268883392935, + "num_queries": 302 + } + }, + "timing": { + "total_retrieval_time_milliseconds": 89819.97203826904, + "indexing_time_milliseconds": 0.0, + "search_time_milliseconds": 89819.97203826904, + "num_queries": 302, + "num_corpus": 1674, + "indexing_throughput_ms_per_doc": 0.0, + "search_throughput_ms_per_query": 297.4171259545333 + }, + "infos": { + "total_retrieval_time_seconds": 89.82 + } + } +} \ No newline at end of file diff --git a/results/pipeline_descriptions/captain-text/description.json b/results/pipeline_descriptions/captain-text/description.json new file mode 100644 index 00000000..8c234aaf --- /dev/null +++ b/results/pipeline_descriptions/captain-text/description.json @@ -0,0 +1,54 @@ +{ + "pipeline_name": "captain-text", + "pipeline_alias": "Captain file-search API (text)", + "description": "Captain (runcaptain.com) managed file-search API. Each corpus page's provided NV-Ingest markdown is indexed as one document ({corpus_id}.md with a '# {doc_id} - page {n}' header); Captain performs server-side chunking, LLM enrichment, and gemini-embedding-2 dense embedding. Retrieval per query: hybrid BM25 + dense with voyage-rerank-2.5 reranking (candidate pool 200, top 100 chunks), pages ranked by best chunk score (MaxP). One API call per query. Closed API: requires CAPTAIN_API_KEY env var (keys at captain.dev).", + "module_path": "pipeline_implementations/captain_text_pipeline.py", + "class_name": "CaptainTextPipeline", + "hardware_information": { + "GPU_type": "none client-side (managed cloud API)", + "num_gpus": 0, + "num_cpu_cores": "n/a (client is a thin HTTP driver, 10 concurrent requests)", + "RAM": "n/a", + "runner_note": "Submitted runs executed from a workstation over WAN to the API in AWS us-east-1; reported search latency includes ~60-80ms/query network overhead. Bulk corpus indexing was performed once ahead of the runs via Captain's S3 batch-ingestion of the same per-page markdown files; the timed index() in the submitted runs therefore verifies the pre-built collection rather than re-ingesting. Measured ingestion from the indexing jobs: finance_en 2,942 pages in 218s as a single job (74 ms/page); the remaining 16,310 pages were indexed by 7 concurrent jobs completing together in ~16.5 min wall (~60 ms/page effective fleet throughput)." + }, + "methodology_notes": { + "runs": "Each dataset was evaluated 3 times with identical configuration; the submitted JSON is the median run by ndcg_cut_10 (retrieval is mildly nondeterministic, about +/-1pt aggregate).", + "tuning": "Configuration (MaxP aggregation, candidate pool 200, semantic-only retrieval for the cross-lingual finance_fr set) was selected on 1,416 self-generated dev queries over the public corpus; ViDoRe benchmark queries were never used for tuning.", + "fallback_integrity": "Every query response was asserted to carry rerank.fallback == null (no silent reranker degradation) from retained raw responses." + }, + "nodes": [ + { + "id": "ingestion_node", + "component_type": "Ingestion", + "model_name": "captain-indexing (chunking + LLM enrichment + gemini-embedding-2)", + "modalities": ["text"], + "description": "Indexes one document per corpus page (provided NV-Ingest markdown) into a per-dataset collection." + }, + { + "id": "retriever_node", + "component_type": "Retriever", + "model_name": "captain hybrid (BM25 + gemini-embedding-2 dense)", + "modalities": ["text"], + "description": "Hybrid retrieval over the page-document collection; returns top chunk candidates." + }, + { + "id": "reranker_node", + "component_type": "Reranker", + "model_name": "voyage-rerank-2.5", + "modalities": ["text"], + "description": "Reranks a 200-candidate pool; final chunk scores are aggregated to pages by MaxP." + } + ], + "edges": [ + { + "source": "ingestion_node", + "target": "retriever_node", + "description": "Indexed page documents form the retrieval corpus." + }, + { + "source": "retriever_node", + "target": "reranker_node", + "description": "Candidate chunks are reranked, then aggregated to page-level rankings." + } + ] +}