diff --git a/.env.example b/.env.example index c5cb5d4..1cae357 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,6 @@ BRAIN_ANSWER_OUT=_brain_answer.txt #SGM_NO_STAR_PROMPT=1 # Folder for that once-per-machine marker (default: $XDG_CONFIG_HOME or ~/.config). #SGM_CONFIG_DIR=~/.config/sqlite-graph-memory + +# PageRank lane in the ordering (ppr.py): RRF with the reranker, top 5 protected. Off by default. +# SGM_PPR=1 diff --git a/CHANGELOG.md b/CHANGELOG.md index a166759..d5f01a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ recorded after the fact rather than backdated to look contemporaneous. ## Unreleased +**The PageRank lane the README credits is now in the package** (#51). `ppr.py` runs +personalized PageRank over the `[[wikilink]]` graph, seeded by the dense hits, and fuses it with +the reranker by RRF while the reranker's top 5 stay where they are. Off by default (`SGM_PPR=1` +or `sgm-ask --ppr`); `eval/run_eval.py --ppr` scores it as its own mode. Any failure returns the +reranker's order. `tests/test_ppr.py` builds a hub note that pushes the reranker's rank-1 hit to +position 11 when the guard is removed, and goes red if it is. + **`sgm-stats `: four health numbers for the link graph.** Orphan rate, links per page, share of pages in the main connected component, and share of concept pages whose `updated` date is older than 90 days, each with an ok/warn/fail verdict and a `--threshold-*` diff --git a/README.md b/README.md index 6e1d1b4..0c51132 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,17 @@ of guessing — 97% of the correct `bridge` answers were *already in the candida cross-encoder was dropping them, because the relation is structural, not textual. Widening retrieval, the obvious fix, would have done nothing at all. +That lane ships in the package as `src/sqlite_graph_memory/ppr.py`, **off by default**: turn it +on with `SGM_PPR=1` or `sgm-ask --ppr`, and score it with `python eval/run_eval.py --gold ... +--ppr`, which adds a third mode next to `vector` and `graph`. The numbers above were measured on +the graph pool, so the matching command is `sgm-ask --graph --ppr`; `--ppr` alone reorders the +vector pool, and `--ab` leaves the lane out so its A/B log stays vector against graph. Any failure in it falls back to +the reranker's order, like the graph hop. Be clear about what that reproduces: the bridge +numbers above came from the maintainer's home vault and home pipeline on 2026-09-16, and they +have not been re-run from the published package. The code is the same algorithm with the same +constants (alpha 0.85, 10 PPR files, top 5 protected), but your vault will give your numbers, +and `bench/` does not score this lane yet. + Seven other changes were rejected by the same ruler, under one rule fixed in advance: **a regression worse than 0.01 nDCG@12 on any class reverts the change.** @@ -334,10 +345,10 @@ Seven other changes were rejected by the same ruler, under one rule fixed in adv | Qwen3-Embedding-0.6B | body Recall 0.767 -> 0.683; reindex 2863 s vs 333 s -> **rejected** | | bge-reranker-v2-m3 | −0.011…−0.018 nDCG on every class except English (+0.041) -> **rejected** | | Qwen3-Reranker-0.6B | title −0.111 nDCG at 12x the cost -> **rejected** | -| BM25 + RRF hybrid | temporal −0.035 -> **merged switched off** | +| BM25 + RRF hybrid | temporal −0.035 -> **merged switched off** (in the home pipeline, not in this package) | | heading-path prefix in every chunk | body −0.023, temporal −0.035 -> **reverted** | | `sqlite-vec` (vec0) | full scan 11.99 s vs 0.14 s for a plain SQLite table, and its ANN missed 1.2 hits of 60 -> vectors live in SQLite, **the extension does not** | -| bi-temporal demotion of superseded notes | temporal +0.130 nDCG on a 41-question temporal set, but title −0.012 -> **merged behind a flag, default off** | +| bi-temporal demotion of superseded notes | temporal +0.130 nDCG on a 41-question temporal set, but title −0.012 -> **merged behind a flag, default off** (in the home pipeline, not in this package) | None of those verdicts is a claim about the models. They are claims about *this corpus with this ruler*, which is the only kind of claim a swap decision actually needs — and the reason diff --git a/eval/run_eval.py b/eval/run_eval.py index d059f08..d827b11 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -9,6 +9,8 @@ vector dense retrieve -> rerank (what you get without the graph) graph dense -> 1-hop [[wikilinks]] -> rerank + ppr graph, then personalized PageRank fused with the reranker by RRF, top 5 + protected (only with --ppr; see src/sqlite_graph_memory/ppr.py) The difference between those two columns is the only honest answer to "does the graph actually help?" — and on our own vault the answer differed by class (see README). @@ -16,6 +18,7 @@ USAGE python eval/run_eval.py --gold eval/gold-2026-09-16.jsonl python eval/run_eval.py --gold ... --limit 20 # smoke run + python eval/run_eval.py --gold ... --ppr # add the PageRank lane as a 3rd mode python eval/run_eval.py --selftest # metrics only, no model, no index Config (env): BRAIN_INDEX_DIR (index to score), TURNSTATE_DB (where --sqlite appends a @@ -119,8 +122,8 @@ def chk(name, got, want): # ------------------------------------------------------------------ runner -def score_gold(gold, progress=True): - """Run every question through both modes. -> (rows, config dict).""" +def score_gold(gold, progress=True, ppr=False): + """Run every question through both modes (three with ppr=True). -> (rows, config dict).""" from sqlite_graph_memory import brain_ask as ba dev = ba.pick_device() @@ -135,15 +138,26 @@ def score_gold(gold, progress=True): by_base = ba.index_by_basename(meta) stem_of = lambda i: Path(meta[i]['path']).stem.lower() - rows, t0 = [], time.time() + rows, t0, ppr_ran = [], time.time(), False for k, g in enumerate(gold): tq = time.time() sims = ba.query_sims(enc, emb, g['query']) base = ba.retrieve_candidates(sims, meta) added = ba.expand_1hop(sims, meta, base, by_base) + lists = {'vector': ba.rerank_candidates(ce, g['query'], base, meta)} + if ppr: + # One rerank of the graph pool serves both modes: `graph` is its top-12, exactly + # what rerank_candidates(topn=12) returns, and `ppr` fuses the FULL list. + full = ba.rerank_candidates(ce, g['query'], base + added, meta, topn=None) + lists['graph'] = full[:TOPN] + lane = ba.ppr_lane(meta, sims, base) + ppr_ran = ppr_ran or bool(lane) + lists['ppr'] = ba.fuse_ppr(full, meta, lane, TOPN) + else: + lists['graph'] = ba.rerank_candidates(ce, g['query'], base + added, meta) res = {} - for mode, cand in (('vector', base), ('graph', base + added)): - ranked = [stem_of(i) for i, _ in ba.rerank_candidates(ce, g['query'], cand, meta)] + for mode, scored in lists.items(): + ranked = [stem_of(i) for i, _ in scored] res[mode] = {'ranked': ranked, 'recall': recall_at(ranked, g['gold']), 'mrr': mrr_at(ranked, g['gold']), @@ -163,6 +177,11 @@ def score_gold(gold, progress=True): sys.stderr.write('!! the graph added no neighbours to ANY question — link resolution or ' 'the index is broken, not the graph. The graph rows below are the ' 'vector rows.\n') + # Same silence, PPR lane: an empty lane falls back to the reranker order, so the ppr rows + # would equal the graph rows and read as "PPR adds nothing" when it never ran. + if ppr and rows and not ppr_ran: + sys.stderr.write('!! the PPR lane returned no files for ANY question (no edges, or it ' + 'failed). The ppr rows below are the graph rows.\n') return rows, cfg @@ -173,7 +192,7 @@ def summarize(rows): groups += [('lang:' + lg, [r for r in rows if r['lang'] == lg]) for lg in ('ru', 'en')] for name, rs in groups: if not rs: continue - for mode in ('vector', 'graph'): + for mode in [m for m in ('vector', 'graph', 'ppr') if m in rs[0]['res']]: out[(name, mode)] = {'n': len(rs), 'recall': sum(r['res'][mode]['recall'] for r in rs) / len(rs), 'mrr': sum(r['res'][mode]['mrr'] for r in rs) / len(rs), @@ -197,6 +216,9 @@ def print_table(summary, cfg, gold_file): if (name, 'graph') in summary: d = summary[(name, 'graph')]['ndcg'] - summary[(name, 'vector')]['ndcg'] print('graph effect on %-9s nDCG@12 %+.3f' % (name, d)) + if (name, 'ppr') in summary: + d = summary[(name, 'ppr')]['ndcg'] - summary[(name, 'graph')]['ndcg'] + print('ppr effect on %-9s nDCG@12 %+.3f (vs graph)' % (name, d)) def write_sqlite(summary, cfg, gold_file, scores_file): @@ -232,6 +254,8 @@ def main(): ap.add_argument('--out', help='scores .jsonl (default: eval/scores-.jsonl)') ap.add_argument('--sqlite', action='store_true', help='also append to gold_eval in TURNSTATE_DB') ap.add_argument('--no-persist', action='store_true', help='print the table, write nothing') + ap.add_argument('--ppr', action='store_true', + help='also score the PageRank lane (graph + PPR/RRF, reranker top 5 protected)') ap.add_argument('--selftest', action='store_true', help='check the metrics and exit') a = ap.parse_args() @@ -256,7 +280,7 @@ def main(): if a.limit: gold = gold[:a.limit] if not gold: sys.exit('gold file %s is empty' % gold_path) - rows, cfg = score_gold(gold) + rows, cfg = score_gold(gold, ppr=a.ppr) summary = summarize(rows) print_table(summary, cfg, a.gold) if os.environ.get('EVAL_MUTANT') == '1': diff --git a/llms.txt b/llms.txt index fc2a818..1ba763a 100644 --- a/llms.txt +++ b/llms.txt @@ -57,7 +57,7 @@ python eval/run_eval.py --gold eval/gold-.jsonl --sqlite ## Configuration (env only) -BRAIN_INDEX_DIR (index folder, default ./index), TURNSTATE_DB (default ./turnstate.db), BRAIN_ANSWER_OUT, BRAIN_AB_JSON (structured A/B output), AB_SOURCE, SGM_NO_STAR_PROMPT=1 (silence the one-time star line after --ab), SGM_CONFIG_DIR. Full list: .env.example. +BRAIN_INDEX_DIR (index folder, default ./index), TURNSTATE_DB (default ./turnstate.db), BRAIN_ANSWER_OUT, BRAIN_AB_JSON (structured A/B output), AB_SOURCE, SGM_NO_STAR_PROMPT=1 (silence the one-time star line after --ab), SGM_CONFIG_DIR, SGM_PPR=1 (PageRank lane in the ordering, off by default). Full list: .env.example. ## Links diff --git a/src/sqlite_graph_memory/brain_ask.py b/src/sqlite_graph_memory/brain_ask.py index b6918d7..06e4f69 100644 --- a/src/sqlite_graph_memory/brain_ask.py +++ b/src/sqlite_graph_memory/brain_ask.py @@ -9,12 +9,15 @@ python brain_ask.py --graph "question" # vector + graph expansion (Associative) python brain_ask.py --ab "question" # run BOTH, diff, log to SQLite (ab_recall) python brain_ask.py --ask "question" # emit a context bundle for an LLM agent + python brain_ask.py --ppr --graph "question" # + PageRank lane in the ORDERING (see ppr.py) Config (env, all optional): BRAIN_INDEX_DIR dir holding the embedding index (default: ./index) TURNSTATE_DB SQLite db for A/B telemetry (default: ./turnstate.db) BRAIN_ANSWER_OUT file to mirror the answer into (default: ./_brain_answer.txt) BRAIN_AB_JSON if set, --ab also writes a structured JSON diff to this path + SGM_PPR=1 same as --ppr: fuse personalized PageRank over [[wikilinks]] with the + reranker by RRF, reranker top 5 protected (off by default) SGM_NO_STAR_PROMPT=1 never print the one-time GitHub star line after --ab SGM_CONFIG_DIR where that line's once-per-machine marker lives (default: $XDG_CONFIG_HOME or ~/.config, /sqlite-graph-memory) @@ -190,12 +193,47 @@ def expand_1hop(sims, meta, base_order, by_base=None, ghops=GHOPS, gmax=GMAX): def rerank_candidates(ce, query, cand, meta, topn=TOPN): - """Cross-encoder rerank of candidate chunks -> [(chunk index, score)], best first.""" + """Cross-encoder rerank of candidate chunks -> [(chunk index, score)], best first. + topn=None returns the whole reranked list (the PPR fusion needs the tail).""" pairs = [(query, meta[i]['title'] + '. ' + meta[i]['snippet']) for i in cand] sc = ce.predict(pairs) if len(pairs) else [] return sorted(zip(cand, sc), key=lambda x: -x[1])[:topn] +def _ppr(): + try: + from sqlite_graph_memory import ppr + except ImportError: # run as a loose script next to ppr.py + import ppr + return ppr + + +def ppr_enabled(env=None): + """True when SGM_PPR asks for the PageRank lane. Never raises.""" + try: + return _ppr().enabled(env) + except Exception: + return False + + +def ppr_lane(meta, sims, base_order): + """PPR file order for this query, or [] (lane off) on ANY failure, including import.""" + try: + return _ppr().ppr_file_order(meta, sims, base_order) + except Exception as e: + sys.stderr.write('PPR lane fell back to reranker order (%s)\n' % str(e)[:140]) + return [] + + +def fuse_ppr(ranked_full, meta, ppr_stems, topn=TOPN): + """Reranker list fused with the PPR order, top `topn`. Any failure: reranker top `topn`.""" + try: + return _ppr().fuse_after_rerank(ranked_full, meta, ppr_stems, topn) + except Exception as e: + sys.stderr.write('PPR fusion fell back to reranker order (%s)\n' % str(e)[:140]) + return list(ranked_full)[:topn] + + STAR_URL = 'https://github.com/tonydzi/sqlite-graph-memory' STAR_MARKER = 'star_prompt_shown' @@ -257,7 +295,7 @@ def main(): args = sys.argv[1:] ask = '--ask' in args graph = '--graph' in args; ab = '--ab' in args - flags = {'--ask', '--graph', '--ab'} + flags = {'--ask', '--graph', '--ab', '--ppr'} query = ' '.join(a for a in args if a not in flags) if not query or not EMB.exists(): sys.stderr.write('need an embedding index (%s) and a query — run index_notes.py first\n' % EMB) @@ -284,10 +322,23 @@ def main(): added = expand_1hop(sims, meta, base_order) graph_order = base_order + added; graph_set = set(added) + # PageRank lane (opt-in): a second ORDERING, not a candidate source. Crash-safe like the + # hop: any failure leaves ppr_stems empty and every list below keeps the reranker's order. + # Not in --ab: the A/B answers "does the 1-hop graph help?", and its rows in ab_recall + # carry no PPR column, so a lane on both arms would be logged as plain vector vs graph. + ppr_stems = [] + if ab and ('--ppr' in args or ppr_enabled()): + sys.stderr.write('PPR lane is not applied in --ab (score it with eval/run_eval.py --ppr)\n') + elif order and ('--ppr' in args or ppr_enabled()): + ppr_stems = ppr_lane(meta, sims, base_order) + # rerank on the matched chunk (precise) ce = load_reranker(dev) def rerank(cand): - return rerank_candidates(ce, query, cand, meta) + if not ppr_stems: + return rerank_candidates(ce, query, cand, meta) + full = rerank_candidates(ce, query, cand, meta, topn=None) # fusion needs the tail + return fuse_ppr(full, meta, ppr_stems, TOPN) # --ab: run BOTH paths in ONE process, diff, log. Answers "does graph REALLY help?" if ab: @@ -349,7 +400,7 @@ def _ser(lst): sys.stderr.write("wiki-graph gated OFF (entity-like query) -> Direct only\n") order = graph_order if (graph and not gated) else base_order reranked = rerank(order) - scope = 'all' + (' /graph+%d' % len(graph_set) if graph else '') + scope = 'all' + (' /graph+%d' % len(graph_set) if graph else '') + (' /ppr' if ppr_stems else '') lines = [f'QUERY: {query}', f'(scope={scope}; e5 {len(order)} files -> rerank; {len(meta)} chunks)', ''] if ask: lines.append('=== CONTEXT BUNDLE (paste to your agent) ===\n') diff --git a/src/sqlite_graph_memory/ppr.py b/src/sqlite_graph_memory/ppr.py new file mode 100644 index 0000000..653b868 --- /dev/null +++ b/src/sqlite_graph_memory/ppr.py @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- +"""ppr.py: personalized PageRank over the [[wikilink]] graph, fused with the reranker by RRF. + +WHY IT EXISTS. On the home vault's frozen gold set (2026-09-16, 60 bridge questions) 97% of +the correct bridge answers were already in the candidate pool, and Recall@12 in that diagnosis +run was 0.225 (the README's 0.251 is the graph-mode baseline of the final measurement): +the cross-encoder dropped them, because the relation is structural, not textual. Adding more +candidates (more hops, larger GMAX) cannot fix that. This module gives the graph a vote in the +ORDERING instead: PPR seeded by the dense hits yields a structural file order, and that order +is fused with the reranker's order by Reciprocal Rank Fusion. Ranks, not scores, because +cosine, cross-encoder logits and PageRank mass are on incomparable scales. + +THE HEAD IS PROTECTED. The first PPR_HEAD (5) positions of the reranker are never touched. When +a query names one thing, the right answer is already first, and any shuffling drops it (home +vault: without the protection title nDCG@12 fell by 0.076; with head=5 it rose by 0.006 at the +same bridge gain). PPR only reorders candidates already in the pool; it never brings new files, +because a file outside the pool has no chunk and no score to show. + +OFF BY DEFAULT. Turn it on with SGM_PPR=1 (or `sgm-ask --ppr`). The home pipeline runs it on by +default; the package does not, because the measurement behind it was taken on one vault. + +CRASH-SAFETY, same contract as the graph hop: no graph, no edges, a PPR failure or a fusion +failure each return the reranker's order unchanged and write one line to stderr. Worst case you +get the old result, never an empty or broken recall. + +COST on the home vault (7.8k files, ~40k edges): building the graph 0.5 s once per process +(cached in memory by an index fingerprint), PPR itself ~3 ms per query. + +Ported from the maintainer's home module brain_graph.py (2026-09-16) for issue #51. +""" +import hashlib +import os +import sys +from pathlib import Path + +import numpy as np + +PPR_ALPHA = 0.85 # damping: share of mass that walks the edges. High on purpose: the gain is + # in moving AWAY from the hits to their neighbours (0.5 gave half the bridge gain) +PPR_ITERS = 20 # power iteration; converges in <10 on a graph this size +PPR_SEED_TOPK = 15 # how many best dense FILES seed the personalization (same as GHOPS) +PPR_TOPM = 10 # how many PPR files enter the fusion. More is WORSE: a long structural tail + # dilutes the top (topm=30 gave bridge +0.053, topm=10 gave +0.103) +PPR_HEAD = 5 # reranker positions the fusion never touches +RRF_K = 60 # RRF constant (Cormack et al. 2009) + +_CACHE = {} # index fingerprint -> Graph, in process memory only + + +def enabled(env=None): + """True when SGM_PPR is set to 1/on/yes/true. Default: off.""" + env = os.environ if env is None else env + return str(env.get('SGM_PPR', '')).strip().lower() in ('1', 'on', 'yes', 'true') + + +def _links_in(path): + # One wikilink parser for the whole package: the one the graph hop uses. + try: + from sqlite_graph_memory.brain_ask import _links_in as f + except ImportError: # run as a loose script next to brain_ask.py + from brain_ask import _links_in as f + return f(path) + + +class Graph: + """File-level wikilink graph in CSR form: node = lowercase file stem.""" + + def __init__(self, stems, idx, indptr, indices): + self.stems = stems + self.idx = idx + self.indptr = indptr + self.indices = indices + self.n_edges = int(len(indices)) + + def ppr(self, seed, alpha=None, iters=None): + """Personalized PageRank: p = alpha * A^T p + (1 - alpha) * seed. + + Dangling nodes (no outgoing links) return their mass to the seed instead of spreading + it over the whole graph, otherwise the unlinked half of a vault would wash out the + personalization. + """ + alpha = PPR_ALPHA if alpha is None else alpha + iters = PPR_ITERS if iters is None else iters + n = len(self.stems) + s = np.asarray(seed, dtype='float64') + tot = s.sum() + if n == 0 or tot <= 0: + return np.zeros(n, dtype='float64') + s = s / tot + if self.n_edges == 0: + return s # no edges: PPR is the personalization itself + p = s.copy() + deg = np.diff(self.indptr) + live = deg > 0 + src = np.repeat(np.arange(n), deg) + for _ in range(iters): + contrib = np.zeros(n, dtype='float64') + contrib[live] = p[live] / deg[live] + nxt = np.zeros(n, dtype='float64') + np.add.at(nxt, self.indices, contrib[src]) + dangling = float(p[~live].sum()) + p = alpha * nxt + (alpha * dangling + (1.0 - alpha)) * s + return p + + +def _fingerprint(meta): + """sha256 over (path, sha) of every chunk, so the cache expires when the index changes.""" + h = hashlib.sha256() + for m in meta: + h.update(str(m.get('path', '')).encode('utf-8', 'replace')) + h.update(b'\x1f') + h.update(str(m.get('sha', '')).encode('utf-8', 'replace')) + h.update(b'\x1e') + return h.hexdigest() + + +def build(meta): + """Build the file graph from the notes' outgoing [[wikilinks]] (self-links and dups dropped). + + A node is a file NAME, because that is what a wikilink addresses: a/index.md and + b/index.md are one node, and the outgoing links of both files become its edges. + """ + try: + fp = _fingerprint(meta) + except Exception: + fp = None + if fp and fp in _CACHE: + return _CACHE[fp] + stems, paths, idx = [], [], {} + for m in meta: + st = Path(m['path']).stem.lower() + if st not in idx: + idx[st] = len(stems) + stems.append(st) + paths.append([]) + if m['path'] not in paths[idx[st]]: + paths[idx[st]].append(m['path']) + indptr = np.zeros(len(stems) + 1, dtype='int64') + indices = [] + for u, files in enumerate(paths): + seen = set() + for p in files: + for tgt in _links_in(p): + v = idx.get(tgt.strip().lower()) + if v is None or v == u or v in seen: + continue + seen.add(v) + indices.append(v) + indptr[u + 1] = len(indices) + g = Graph(stems, idx, indptr, np.asarray(indices, dtype='int64')) + if fp: + _CACHE[fp] = g + return g + + +def ppr_file_order(meta, sims, base_order, graph=None, topm=None, seed_topk=None): + """File stems ordered by personalized PageRank, seeded by the dense hits in base_order. + + Seed weight = best chunk cosine + 1, so it is non-negative (PageRank needs a non-negative + teleport, cosine lives in [-1, 1]). Returns [] on any failure; the caller keeps its order. + """ + topm = PPR_TOPM if topm is None else topm + seed_topk = PPR_SEED_TOPK if seed_topk is None else seed_topk + try: + g = build(meta) if graph is None else graph + if not g.stems or g.n_edges == 0: + return [] + seed = np.zeros(len(g.stems), dtype='float64') + seen, used = set(), 0 + for i in base_order: + st = Path(meta[i]['path']).stem.lower() + if st in seen: + continue + seen.add(st) + u = g.idx.get(st) + if u is None: + continue + seed[u] = max(seed[u], float(sims[i]) + 1.0) # seed starts at 0: never negative + used += 1 + if used >= seed_topk: + break + if seed.sum() <= 0: + return [] + p = g.ppr(seed) + top = np.argsort(-p, kind='stable')[:max(1, topm)] + return [g.stems[int(u)] for u in top if p[int(u)] > 0] + except Exception as e: + sys.stderr.write('PPR lane failed (%s) -> reranker order unchanged\n' % str(e)[:140]) + return [] + + +def rrf_fuse(*ranked_lists, k=RRF_K): + """Reciprocal Rank Fusion: score(d) = sum 1 / (k + rank_d + 1). Ties break by item value.""" + score, order = {}, [] + for lst in ranked_lists: + for rank, i in enumerate(lst): + if i not in score: + score[i] = 0.0 + order.append(i) + score[i] += 1.0 / (k + rank + 1) + order.sort(key=lambda i: (-score[i], i)) + return order + + +def fuse_after_rerank(ranked_idx, meta, ppr_stems, topn, head=None): + """Pipeline hook: reorder [(chunk index, score)] from the reranker, then cut to topn. + + `ranked_idx` must be the FULL reranked list, not the top-12: the point is to lift the right + neighbour out of the tail, which a fusion that only sees the top-12 cannot do. Empty PPR + returns the first topn exactly as given. + + Every item is kept: two notes with the same file name in different folders are one graph + node (wikilinks address notes by name) but stay two results, so the output is never + shorter than the input. Positions, not names, go into RRF, so a tie breaks toward the + reranker's earlier position instead of the alphabetical order of file names. + """ + if not ppr_stems or not ranked_idx: + return list(ranked_idx)[:topn] + try: + items = list(ranked_idx) + head = PPR_HEAD if head is None else head + stems = [Path(meta[i]['path']).stem.lower() for i, _ in items] + kept = set(stems[:head]) + first = {} + for p in range(head, len(items)): + first.setdefault(stems[p], p - head) + ppr_pos = [first[s] for s in ppr_stems if s in first and s not in kept] + if not ppr_pos: + return items[:topn] + fused = rrf_fuse(list(range(len(items) - head)), ppr_pos) + return (items[:head] + [items[head + p] for p in fused])[:topn] + except Exception as e: + sys.stderr.write('PPR fusion failed (%s) -> reranker order\n' % str(e)[:140]) + return list(ranked_idx)[:topn] diff --git a/tests/test_ppr.py b/tests/test_ppr.py new file mode 100644 index 0000000..bef8aa6 --- /dev/null +++ b/tests/test_ppr.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +"""The PageRank lane: what it moves, what it must never move, and that it fails closed. + +The protection test is the one that matters. A hub note (everything links to it) and its +neighbours get the most PageRank mass; without the top-5 guard, RRF lifts ten of them over the +reranker's rank-1 hit and pushes it to position 11. Remove the guard and +`test_hub_cannot_push_reranker_rank1_out_of_top5` goes red. + +No model download, no network: real wikilink files, a hand-made similarity vector, and a fake +cross-encoder. +""" +import sys +from pathlib import Path + +import numpy as np +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'src')) +sys.path.insert(0, str(ROOT / 'eval')) + +from sqlite_graph_memory import brain_ask as ba # noqa: E402 +from sqlite_graph_memory import ppr # noqa: E402 + + +@pytest.fixture(autouse=True) +def _fresh_cache(): + ppr._CACHE.clear() + yield + ppr._CACHE.clear() + + +def _vault(tmp_path, links): + """links: {stem: [targets]} -> meta list with one chunk per note, in dict order.""" + meta = [] + for stem, tgts in links.items(): + p = tmp_path / ('%s.md' % stem) + p.write_text('# %s\n\n' % stem + ' '.join('[[%s]]' % t for t in tgts) + '\n', + encoding='utf-8') + meta.append({'path': str(p), 'title': stem, 'snippet': stem, 'sha': stem}) + return meta + + +def _hub_vault(tmp_path): + """15 seed notes all link to `hub` and to c00..c09; hub links to the cluster too. + `answer` is the reranker's rank-1 but only dense rank 16, so it is not a PPR seed. + 20 filler notes link nowhere.""" + cluster = ['c%02d' % i for i in range(10)] + links = {} + for i in range(15): + links['seed%02d' % i] = ['hub'] + cluster + links['answer'] = [] + links['hub'] = cluster + for c in cluster: + links[c] = ['hub'] + for i in range(20): + links['filler%02d' % i] = [] + meta = _vault(tmp_path, links) + stems = [m['title'] for m in meta] + sims = np.zeros(len(meta)) + for k, st in enumerate(stems): # dense: seeds first, then answer, then the rest + sims[k] = 0.9 - 0.01 * k if st.startswith('seed') else 0.3 + sims[stems.index('answer')] = 0.5 + base_order = sorted(range(len(meta)), key=lambda i: -sims[i]) + # The reranker: answer first, then fillers, then the hub cluster at the tail. + rr = (['answer'] + ['filler%02d' % i for i in range(20)] + ['seed%02d' % i for i in range(15)] + + cluster + ['hub']) + ranked_full = [(stems.index(s), float(len(rr) - r)) for r, s in enumerate(rr)] + return meta, sims, base_order, ranked_full, stems + + +# ------------------------------------------------------------------ the guard +def test_hub_cannot_push_reranker_rank1_out_of_top5(tmp_path): + meta, sims, base_order, ranked_full, stems = _hub_vault(tmp_path) + lane = ppr.ppr_file_order(meta, sims, base_order) + assert lane[0] == 'hub' # the hub really is the PPR favourite + assert 'answer' not in lane # and rank-1 gets no PPR help + out = [meta[i]['title'] for i, _ in ppr.fuse_after_rerank(ranked_full, meta, lane, 12)] + rr_top5 = [meta[i]['title'] for i, _ in ranked_full[:5]] + assert out[0] == 'answer' + assert out[:5] == rr_top5 # the head is left exactly as the reranker had it + + +def test_without_the_guard_the_hub_cluster_does_displace_rank1(tmp_path): + # The scenario above is a real threat, not a strawman: head=0 shows what the guard stops. + meta, sims, base_order, ranked_full, _ = _hub_vault(tmp_path) + lane = ppr.ppr_file_order(meta, sims, base_order) + out = [meta[i]['title'] for i, _ in ppr.fuse_after_rerank(ranked_full, meta, lane, 12, head=0)] + assert 'answer' not in out[:5] + + +def test_ppr_lifts_a_tail_neighbour_into_the_top12(tmp_path): + meta, sims, base_order, ranked_full, _ = _hub_vault(tmp_path) + before = [meta[i]['title'] for i, _ in ranked_full[:12]] + lane = ppr.ppr_file_order(meta, sims, base_order) + after = [meta[i]['title'] for i, _ in ppr.fuse_after_rerank(ranked_full, meta, lane, 12)] + assert 'hub' not in before and 'hub' in after + assert len(after) == 12 and len(set(after)) == 12 + + +def test_fusion_never_brings_files_from_outside_the_pool(): + meta = [{'path': '/v/%s.md' % s} for s in 'abcdef'] + ranked = [(i, 1.0) for i in range(6)] + out = ppr.fuse_after_rerank(ranked, meta, ['zzz', 'f', 'e'], 12, head=2) + assert sorted(i for i, _ in out) == list(range(6)) + assert [i for i, _ in out][:2] == [0, 1] + + +def test_same_name_in_two_folders_stays_two_results(): + # Wikilinks address notes by name, so a/x.md and b/x.md are one graph node, but they are + # two results and the fused list must not be shorter than the reranker's. + meta = [{'path': '/v/%s.md' % s} for s in ('h0', 'h1', 'a/x', 'b/x', 'y', 'z')] + ranked = [(i, float(-i)) for i in range(6)] + out = ppr.fuse_after_rerank(ranked, meta, ['x', 'z'], 12, head=2) + assert sorted(i for i, _ in out) == list(range(6)) + assert len(ppr.fuse_after_rerank(ranked, meta, ['x'], 5, head=2)) == 5 + + +def test_second_file_with_the_same_name_contributes_its_links(tmp_path): + (tmp_path / 'a').mkdir(); (tmp_path / 'b').mkdir() + meta = [] + for rel, body in (('a/index.md', '[[x]]'), ('b/index.md', '[[y]]'), ('x.md', ''), ('y.md', '')): + f = tmp_path / rel + f.write_text(body + '\n', encoding='utf-8') + meta.append({'path': str(f), 'sha': rel}) + g = ppr.build(meta) + u = g.idx['index'] + out = {g.stems[int(v)] for v in g.indices[g.indptr[u]:g.indptr[u + 1]]} + assert out == {'x', 'y'} + + +def test_empty_lane_equals_the_graph_mode_exactly(): + # The ppr column must not differ from graph when PPR has nothing to say, or the printed + # "ppr effect" would credit the lane with something it did not do. + meta = [{'path': '/v/%s.md' % s} for s in ('a/index', 'b/index', 'c', 'd')] + ranked = [(0, 5.0), (1, 4.0), (2, 3.0), (3, 2.0)] + assert ppr.fuse_after_rerank(ranked, meta, [], 12) == ranked + assert ba.fuse_ppr(ranked, meta, [], 12) == ranked + assert ppr.fuse_after_rerank(ranked, meta, ['c', 'd', 'index'], 12) == ranked + assert sorted(ppr.fuse_after_rerank(ranked, meta, ['d', 'c'], 12, head=0)) == sorted(ranked) + + +# ------------------------------------------------------------------ off by default +@pytest.mark.parametrize('env,want', [({}, False), ({'SGM_PPR': ''}, False), + ({'SGM_PPR': '0'}, False), ({'SGM_PPR': '1'}, True), + ({'SGM_PPR': 'on'}, True)]) +def test_lane_is_off_unless_asked(env, want): + assert ppr.enabled(env) is want + assert ba.ppr_enabled(env) is want + + +# ------------------------------------------------------------------ crash-safety +def test_graph_without_edges_returns_the_reranker_order(tmp_path): + meta = _vault(tmp_path, {'a': [], 'b': [], 'c': []}) + lane = ppr.ppr_file_order(meta, np.array([0.9, 0.5, 0.1]), [0, 1, 2]) + assert lane == [] + ranked = [(2, 3.0), (0, 2.0), (1, 1.0)] + assert ppr.fuse_after_rerank(ranked, meta, lane, 12) == ranked + + +def test_broken_meta_degrades_to_empty_lane(capsys): + assert ppr.ppr_file_order([{'no_path': 1}], np.array([1.0]), [0]) == [] + assert 'reranker order unchanged' in capsys.readouterr().err + + +def test_broken_fusion_returns_reranker_top_n(capsys): + ranked = [(0, 2.0), (1, 1.0)] + assert ppr.fuse_after_rerank(ranked, [{'nope': 1}], ['x'], 1) == [(0, 2.0)] + assert 'reranker order' in capsys.readouterr().err + + +def test_brain_ask_hooks_survive_a_missing_module(monkeypatch, capsys): + def boom(): + raise ImportError('no ppr module') + monkeypatch.setattr(ba, '_ppr', boom) + ranked = [(i, float(-i)) for i in range(20)] + assert ba.ppr_lane([], np.array([]), []) == [] + assert ba.fuse_ppr(ranked, [], ['x'], 12) == ranked[:12] + assert ba.ppr_enabled({'SGM_PPR': '1'}) is False + assert 'fell back' in capsys.readouterr().err + + +# ------------------------------------------------------------------ the graph mode is untouched +class _FakeCE: + def predict(self, pairs): + return [len(t) % 7 + 0.001 * k for k, (_, t) in enumerate(pairs)] + + +def test_full_rerank_cut_at_12_equals_the_graph_mode(): + meta = [{'title': 't%d' % i * (i % 5 + 1), 'snippet': 's'} for i in range(40)] + cand = list(range(40)) + full = ba.rerank_candidates(_FakeCE(), 'q', cand, meta, topn=None) + assert len(full) == 40 + assert full[:12] == ba.rerank_candidates(_FakeCE(), 'q', cand, meta) + + +def test_summarize_reports_ppr_as_its_own_mode(): + from run_eval import summarize + r = {'recall': 1.0, 'mrr': 1.0, 'ndcg': 1.0} + rows = [{'cls': 'bridge', 'lang': 'en', 'res': {'vector': r, 'graph': r, 'ppr': r}}] + assert ('bridge', 'ppr') in summarize(rows) + rows[0]['res'].pop('ppr') + assert ('bridge', 'ppr') not in summarize(rows)