EdgarBabajanyan · Sep 2, 2026
diff --git a/pipeline_implementations/captain_text_pipeline.py b/pipeline_implementations/captain_text_pipeline.py
@@ -0,0 +1,113 @@
+"""Captain (runcaptain.com) managed file-search API — text pipeline.
+
+Ingestion: each ViDoRe v3 corpus page's provided NV-Ingest markdown is indexed
+as one document named `{corpus_id}.md`, prefixed with a one-line
+`# {doc_id} - page {n}` header so chunks carry document identity. Captain then
+does its own chunking, LLM enrichment, and embedding server-side.
+
+Retrieval: hybrid BM25 + dense retrieval with voyage-rerank-2.5 reranking
+(candidate pool 200), one API call per query. Pages are ranked by their best
+chunk score (MaxP); returned document filenames map back to corpus ids.
+
+Requires the CAPTAIN_API_KEY environment variable (closed API; free keys at
+https://www.captain.dev). See description.json for run/timing details.
+"""
+import json
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import requests
+
+from vidore_benchmark.pipeline_evaluation.base_pipeline import BasePipeline
+
+API = os.environ.get("CAPTAIN_API", "https://api.runcaptain.com")
+KEY = os.environ["CAPTAIN_API_KEY"]
+EPS = 1e-6
+
+
+class CaptainTextPipeline(BasePipeline):
+    def __init__(self, top_k: int = 100, workers: int = 10, **kwargs):
+        self.top_k = top_k
+        self.workers = workers
+        self.s = requests.Session()
+        self.s.headers.update({"Authorization": f"Bearer {KEY}",
+                               "Content-Type": "application/json"})
+        self.collection = None
+
+    # ------------------------------------------------------------------ index
+    def index(self, corpus_ids, corpus_images, corpus_texts, dataset_name=None):
+        ds = (dataset_name or "vidore_v3").rsplit("vidore_v3_", 1)[-1]
+        self.collection = f"vidore-{ds}-text"
+        r = self.s.put(f"{API}/v2/collections/{self.collection}", json={}, timeout=60)
+        if r.status_code == 409 or (r.status_code == 200 and self._doc_count() >= len(corpus_ids)):
+            return  # already indexed (pre-built corpus reused across runs)
+        if r.status_code not in (200, 201):
+            r.raise_for_status()
+
+        def _put(pair):
+            cid, text = pair
+            body = {"content": text or " ", "file_name": f"{cid}.md"}
+            for attempt in range(3):
+                resp = self.s.post(f"{API}/v2/collections/{self.collection}/index/text",
+                                   json=body, timeout=120)
+                if resp.status_code in (200, 201, 202):
+                    return
+                time.sleep(2 * (attempt + 1))
+            resp.raise_for_status()
+
+        with ThreadPoolExecutor(max_workers=self.workers) as ex:
+            list(ex.map(_put, zip(corpus_ids, corpus_texts)))
+        # poll until documents are queryable
+        deadline = time.time() + 3600
+        while time.time() < deadline and self._doc_count() < len(corpus_ids):
+            time.sleep(15)
+
+    def _doc_count(self):
+        try:
+            r = self.s.get(f"{API}/v2/collections/{self.collection}/documents?limit=1",
+                           timeout=60)
+            return (r.json().get("pagination") or {}).get("total") or \
+                len(r.json().get("documents") or [])
+        except Exception:
+            return 0
+
+    # ----------------------------------------------------------------- search
+    def _one(self, qid, query):
+        body = {"query": query, "limit": 100,
+                "rerank": {"enabled": True, "candidate_limit": 200},
+                "include": {"document": True, "metadata": False}}
+        if self.collection.endswith("finance_fr-text"):
+            # a-priori rule for the cross-lingual set (English queries, French
+            # corpus): keyword retrieval contributes nothing, use dense only.
+            body["semantic_ratio"] = 1.0
+        for attempt in range(3):
+            try:
+                r = self.s.post(f"{API}/v3/collections/{self.collection}/query",
+                                json=body, timeout=120)
+                r.raise_for_status()
+                break
+            except Exception:
+                if attempt == 2:
+                    raise
+                time.sleep(3 * (attempt + 1))
+        d = r.json()
+        assert not (d.get("rerank") or {}).get("fallback"), "rerank fallback — rerun"
+        agg = {}
+        for res in d.get("results", []):
+            fname = ((res.get("document") or {}).get("filename") or "").rsplit("/", 1)[-1]
+            stem = fname[:-3] if fname.endswith(".md") else fname
+            if not stem.isdigit():
+                continue
+            score = float(res.get("score") or 0.0)
+            agg[stem] = max(agg.get(stem, 0.0), score)   # MaxP page aggregation
+        ranked = sorted(agg.items(), key=lambda kv: -kv[1])[: self.top_k]
+        return qid, {cid: s - i * EPS for i, (cid, s) in enumerate(ranked)}
+
+    def search(self, query_ids, queries):
+        out = {}
+        t0 = time.time()
+        with ThreadPoolExecutor(max_workers=self.workers) as ex:
+            for qid, run in ex.map(lambda a: self._one(*a), zip(query_ids, queries)):
+                out[str(qid)] = run
+        return out, {"total_retrieval_time_seconds": round(time.time() - t0, 2)}
diff --git a/results/metrics/captain-text/vidore_v3_computer_science.json b/results/metrics/captain-text/vidore_v3_computer_science.json
@@ -0,0 +1,70 @@
+{
+  "dataset": "vidore/vidore_v3_computer_science",
+  "split": "test",
+  "language": "english",
+  "pipeline_type": null,
+  "module_path": "captain_official.py",
+  "class_name": "CaptainTextOfficial",
+  "pipeline_args": {},
+  "aggregated_metrics": {
+    "overall": {
+      "map": 0.7895771479970486,
+      "recip_rank": 0.9343466223698782,
+      "P_1": 0.8930232558139535,
+      "P_5": 0.5469767441860466,
+      "P_10": 0.35953488372093023,
+      "P_20": 0.2111627906976744,
+      "recall_1": 0.3411533815022187,
+      "recall_5": 0.7229698854117459,
+      "recall_10": 0.8468431697501465,
+      "recall_20": 0.9193194789706417,
+      "recall_50": 0.9434451723986608,
+      "recall_100": 0.9521790225278597,
+      "ndcg_cut_1": 0.8348837209302326,
+      "ndcg_cut_5": 0.8248622688551305,
+      "ndcg_cut_10": 0.8411160239742446,
+      "ndcg_cut_20": 0.8648934625660156,
+      "ndcg_cut_100": 0.8761385960959897,
+      "map_cut_1": 0.3411533815022187,
+      "map_cut_10": 0.7474295024271491,
+      "map_cut_100": 0.7895771479970486
+    },
+    "by_language": {
+      "english": {
+        "map": 0.7895771479970486,
+        "recip_rank": 0.9343466223698782,
+        "P_1": 0.8930232558139535,
+        "P_5": 0.5469767441860466,
+        "P_10": 0.35953488372093023,
+        "P_20": 0.2111627906976744,
+        "recall_1": 0.3411533815022187,
+        "recall_5": 0.7229698854117459,
+        "recall_10": 0.8468431697501465,
+        "recall_20": 0.9193194789706417,
+        "recall_50": 0.9434451723986608,
+        "recall_100": 0.9521790225278597,
+        "ndcg_cut_1": 0.8348837209302326,
+        "ndcg_cut_5": 0.8248622688551305,
+        "ndcg_cut_10": 0.8411160239742446,
+        "ndcg_cut_20": 0.8648934625660156,
+        "ndcg_cut_100": 0.8761385960959897,
+        "map_cut_1": 0.3411533815022187,
+        "map_cut_10": 0.7474295024271491,
+        "map_cut_100": 0.7895771479970486,
+        "num_queries": 215
+      }
+    },
+    "timing": {
+      "total_retrieval_time_milliseconds": 102711.82107925415,
+      "indexing_time_milliseconds": 0.0,
+      "search_time_milliseconds": 102711.82107925415,
+      "num_queries": 215,
+      "num_corpus": 1360,
+      "indexing_throughput_ms_per_doc": 0.0,
+      "search_throughput_ms_per_query": 477.72940036862394
+    },
+    "infos": {
+      "total_retrieval_time_seconds": 102.71
+    }
+  }
+}
diff --git a/results/metrics/captain-text/vidore_v3_energy.json b/results/metrics/captain-text/vidore_v3_energy.json
@@ -0,0 +1,70 @@
+{
+  "dataset": "vidore/vidore_v3_energy",
+  "split": "test",
+  "language": "english",
+  "pipeline_type": null,
+  "module_path": "captain_official.py",
+  "class_name": "CaptainTextOfficial",
+  "pipeline_args": {},
+  "aggregated_metrics": {
+    "overall": {
+      "map": 0.6536825354885418,
+      "recip_rank": 0.7953532184948794,
+      "P_1": 0.7045454545454546,
+      "P_5": 0.30714285714285716,
+      "P_10": 0.20194805194805196,
+      "P_20": 0.12224025974025973,
+      "recall_1": 0.4144982449265048,
+      "recall_5": 0.6901633096337426,
+      "recall_10": 0.7840659496211037,
+      "recall_20": 0.8482336284089727,
+      "recall_50": 0.8927673748322279,
+      "recall_100": 0.8936360060375704,
+      "ndcg_cut_1": 0.672077922077922,
+      "ndcg_cut_5": 0.6981245103225042,
+      "ndcg_cut_10": 0.7256963666681007,
+      "ndcg_cut_20": 0.7443411803975708,
+      "ndcg_cut_100": 0.7580956424452949,
+      "map_cut_1": 0.4144982449265048,
+      "map_cut_10": 0.6276145516677658,
+      "map_cut_100": 0.6536825354885418
+    },
+    "by_language": {
+      "english": {
+        "map": 0.6536825354885418,
+        "recip_rank": 0.7953532184948794,
+        "P_1": 0.7045454545454546,
+        "P_5": 0.30714285714285716,
+        "P_10": 0.20194805194805196,
+        "P_20": 0.12224025974025973,
+        "recall_1": 0.4144982449265048,
+        "recall_5": 0.6901633096337426,
+        "recall_10": 0.7840659496211037,
+        "recall_20": 0.8482336284089727,
+        "recall_50": 0.8927673748322279,
+        "recall_100": 0.8936360060375704,
+        "ndcg_cut_1": 0.672077922077922,
+        "ndcg_cut_5": 0.6981245103225042,
+        "ndcg_cut_10": 0.7256963666681007,
+        "ndcg_cut_20": 0.7443411803975708,
+        "ndcg_cut_100": 0.7580956424452949,
+        "map_cut_1": 0.4144982449265048,
+        "map_cut_10": 0.6276145516677658,
+        "map_cut_100": 0.6536825354885418,
+        "num_queries": 308
+      }
+    },
+    "timing": {
+      "total_retrieval_time_milliseconds": 200609.8837852478,
+      "indexing_time_milliseconds": 0.0,
+      "search_time_milliseconds": 200609.8837852478,
+      "num_queries": 308,
+      "num_corpus": 2225,
+      "indexing_throughput_ms_per_doc": 0.0,
+      "search_throughput_ms_per_query": 651.3307915105448
+    },
+    "infos": {
+      "total_retrieval_time_seconds": 200.61
+    }
+  }
+}
diff --git a/results/metrics/captain-text/vidore_v3_finance_en.json b/results/metrics/captain-text/vidore_v3_finance_en.json
@@ -0,0 +1,70 @@
+{
+  "dataset": "vidore/vidore_v3_finance_en",
+  "split": "test",
+  "language": "english",
+  "pipeline_type": null,
+  "module_path": "captain_official.py",
+  "class_name": "CaptainTextOfficial",
+  "pipeline_args": {},
+  "aggregated_metrics": {
+    "overall": {
+      "map": 0.6202737162091443,
+      "recip_rank": 0.8237355127135183,
+      "P_1": 0.7249190938511327,
+      "P_5": 0.39158576051779936,
+      "P_10": 0.254368932038835,
+      "P_20": 0.15436893203883495,
+      "recall_1": 0.3263296696368024,
+      "recall_5": 0.6268750240355939,
+      "recall_10": 0.7321575400123749,
+      "recall_20": 0.8142189300947454,
+      "recall_50": 0.8884690862789092,
+      "recall_100": 0.8987406269830598,
+      "ndcg_cut_1": 0.6844660194174758,
+      "ndcg_cut_5": 0.676064451859597,
+      "ndcg_cut_10": 0.6956430294972429,
+      "ndcg_cut_20": 0.7208042805404912,
+      "ndcg_cut_100": 0.7498745601698638,
+      "map_cut_1": 0.3263296696368024,
+      "map_cut_10": 0.582357619891026,
+      "map_cut_100": 0.6202737162091443
+    },
+    "by_language": {
+      "english": {
+        "map": 0.6202737162091443,
+        "recip_rank": 0.8237355127135183,
+        "P_1": 0.7249190938511327,
+        "P_5": 0.39158576051779936,
+        "P_10": 0.254368932038835,
+        "P_20": 0.15436893203883495,
+        "recall_1": 0.3263296696368024,
+        "recall_5": 0.6268750240355939,
+        "recall_10": 0.7321575400123749,
+        "recall_20": 0.8142189300947454,
+        "recall_50": 0.8884690862789092,
+        "recall_100": 0.8987406269830598,
+        "ndcg_cut_1": 0.6844660194174758,
+        "ndcg_cut_5": 0.676064451859597,
+        "ndcg_cut_10": 0.6956430294972429,
+        "ndcg_cut_20": 0.7208042805404912,
+        "ndcg_cut_100": 0.7498745601698638,
+        "map_cut_1": 0.3263296696368024,
+        "map_cut_10": 0.582357619891026,
+        "map_cut_100": 0.6202737162091443,
+        "num_queries": 309
+      }
+    },
+    "timing": {
+      "total_retrieval_time_milliseconds": 187896.84796333313,
+      "indexing_time_milliseconds": 0.0,
+      "search_time_milliseconds": 187896.84796333313,
+      "num_queries": 309,
+      "num_corpus": 2942,
+      "indexing_throughput_ms_per_doc": 0.0,
+      "search_throughput_ms_per_query": 608.0804141208192
+    },
+    "infos": {
+      "total_retrieval_time_seconds": 187.89
+    }
+  }
+}