From f43b17502ef58e16044de5565d7e01657e19e7ff Mon Sep 17 00:00:00 2001 From: Anton Dziatkovskii <194927794+tonydzi@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:26:20 -0700 Subject: [PATCH 1/2] fix(index): skip dot-files, not only dot-directories (closes #25) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit iter_notes filtered hidden directories by checking rel.parts[:-1] — every path component except the last, which is the filename. So .draft.md or .trash.md in the vault root, or in any visible folder, was indexed as a real note while the same file one level down inside .drafts/ was skipped. Reproduced 2026-10-13 with .secret.md next to ok.md: both came back. The filter now checks every component. BRAIN_INDEX_HIDDEN=1 still admits both kinds of hidden path, since someone legitimately keeps notes in one. tests/test_index_notes_dotfiles.py pins all four cases; shown red on the old line (2 of 4 fail, exactly the two file cases) before the change. Full suite: 102 passed. Assisted-by: Claude Code / claude-fable-5-1 Machine: MacBook-Anton Account: dzyatkovskiy.a2@gmail.com Operator: anton Co-Authored-By: Claude Fable 5.1 --- src/sqlite_graph_memory/index_notes.py | 6 +-- tests/test_index_notes_dotfiles.py | 63 ++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/test_index_notes_dotfiles.py diff --git a/src/sqlite_graph_memory/index_notes.py b/src/sqlite_graph_memory/index_notes.py index 0021908..7d66634 100644 --- a/src/sqlite_graph_memory/index_notes.py +++ b/src/sqlite_graph_memory/index_notes.py @@ -77,15 +77,15 @@ def fm_date(fm): def iter_notes(notes_dir): """Files that count as a note under ``notes_dir``. - Skips dot-directories and Syncthing conflict copies. Set ``BRAIN_INDEX_HIDDEN=1`` - to index hidden folders too — someone will legitimately keep notes in one. + Skips dot-directories, dot-files (``.draft.md``) and Syncthing conflict copies. + Set ``BRAIN_INDEX_HIDDEN=1`` to index hidden paths too — someone will legitimately keep notes in one. """ root = Path(notes_dir) include_hidden = os.getenv('BRAIN_INDEX_HIDDEN', '') not in ('', '0', 'false', 'no') out = [] for p in root.rglob('*.md'): rel = p.relative_to(root) - if not include_hidden and any(part.startswith('.') for part in rel.parts[:-1]): + if not include_hidden and any(part.startswith('.') for part in rel.parts): continue if CONFLICT_MARKER in p.name: continue diff --git a/tests/test_index_notes_dotfiles.py b/tests/test_index_notes_dotfiles.py new file mode 100644 index 0000000..d5d49c5 --- /dev/null +++ b/tests/test_index_notes_dotfiles.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +"""Dot-FILES are skipped the same way dot-DIRECTORIES are (issue #25). + +`iter_notes` filtered hidden *directories* by checking `rel.parts[:-1]` — every +path component except the last. The last component is the filename, so a note +called `.draft.md` or `.trash.md` sitting in the vault root (or in any visible +folder) was indexed as if it were a real note, while the same file one level +down inside `.drafts/` was correctly skipped. Reproduced 2026-10-13 with +`.secret.md` next to `ok.md`: `iter_notes()` returned both. + +RED-FIRST: restore `rel.parts[:-1]` on the filter line and +`test_dot_file_in_root_is_skipped` fails with `.draft.md` in the result. + +Both halves are pinned on purpose — the directory case already worked and must +keep working, the file case is the fix — and `BRAIN_INDEX_HIDDEN=1` must still +let both through, because someone legitimately keeps notes in a hidden folder. + +No model download, no network: pure file work. +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from sqlite_graph_memory.index_notes import iter_notes # noqa: E402 + + +def _vault(tmp_path: Path) -> Path: + (tmp_path / "ok.md").write_text("visible note", encoding="utf-8") + (tmp_path / ".draft.md").write_text("hidden file in root", encoding="utf-8") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / ".trash.md").write_text("hidden file in visible dir", encoding="utf-8") + (tmp_path / ".drafts").mkdir() + (tmp_path / ".drafts" / "inside.md").write_text("visible name, hidden dir", encoding="utf-8") + return tmp_path + + +def _names(root: Path) -> set[str]: + return {str(p.relative_to(root)) for p in iter_notes(root)} + + +def test_dot_file_in_root_is_skipped(tmp_path, monkeypatch): + monkeypatch.delenv("BRAIN_INDEX_HIDDEN", raising=False) + assert ".draft.md" not in _names(_vault(tmp_path)) + + +def test_dot_file_in_visible_dir_is_skipped(tmp_path, monkeypatch): + monkeypatch.delenv("BRAIN_INDEX_HIDDEN", raising=False) + assert "sub/.trash.md" not in _names(_vault(tmp_path)) + + +def test_dot_dir_still_skipped_and_visible_note_still_kept(tmp_path, monkeypatch): + """The pre-existing behaviour must survive the fix.""" + monkeypatch.delenv("BRAIN_INDEX_HIDDEN", raising=False) + names = _names(_vault(tmp_path)) + assert ".drafts/inside.md" not in names + assert "ok.md" in names + + +def test_hidden_flag_admits_dot_files_and_dot_dirs(tmp_path, monkeypatch): + """Opt-in must cover both kinds of hidden path, not just directories.""" + monkeypatch.setenv("BRAIN_INDEX_HIDDEN", "1") + names = _names(_vault(tmp_path)) + assert {"ok.md", ".draft.md", "sub/.trash.md", ".drafts/inside.md"} <= names From 0670d999fcae4066b4e5dd22f40e2bb285053309 Mon Sep 17 00:00:00 2001 From: Vardhman Gupta Date: Tue, 20 Oct 2026 15:58:34 +0530 Subject: [PATCH 2/2] fix(eval): build_gold uses iter_notes; dot-root vault test; docs for #25 - eval/build_gold.py read_notes() walked the vault with its own copy of the filter (rel.parts[:-1]) and ignored BRAIN_INDEX_HIDDEN, so a root-level .draft.md could become a gold question while the indexer skipped it. It now calls sqlite_graph_memory.index_notes.iter_notes: one rule, one place. - tests: _names() uses as_posix() so the forward-slash assertions hold on Windows; new test pins a vault whose root is itself a dot-directory (guards against filtering p.parts instead of rel.parts); new test_eval case proves build_gold drops .draft.md and honours BRAIN_INDEX_HIDDEN=1. - docs: index_notes docstring, examples/README env table and CHANGELOG say dot-files and dot-directories. Assisted-by: Claude Code / claude-opus-5-5 Machine: MacBook-Anton Account: dzyatkovskiy.a2@gmail.com Operator: anton Co-Authored-By: Claude Opus 5.5 # Conflicts: # CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ eval/build_gold.py | 9 ++++++--- examples/README.md | 2 +- src/sqlite_graph_memory/index_notes.py | 6 +++--- tests/test_eval.py | 19 +++++++++++++++++++ tests/test_index_notes_dotfiles.py | 20 +++++++++++++++++++- 6 files changed, 56 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a166759..1f5002a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ recorded after the fact rather than backdated to look contemporaneous. ## Unreleased +**Hidden note files are skipped like hidden folders.** A note whose filename starts with a +dot, such as `.ideas.md` or `.draft.md`, used to be indexed when it sat in the vault root or a +visible folder, while the same file inside `.drafts/` was skipped. Now both are skipped, so +after re-indexing those notes stop showing up in answers. `BRAIN_INDEX_HIDDEN=1` brings back +the old behaviour and indexes hidden files and folders alike. `eval/build_gold.py` now walks the +vault with the indexer's own filter, so it follows the same rule and the same variable instead +of keeping its own copy (issue #25). + **`sgm-stats `: four health numbers for the link graph.** Orphan rate, links per page, share of pages in the main connected component, and share of concept pages whose `updated` date is older than 90 days, each with an ok/warn/fail verdict and a `--threshold-*` diff --git a/eval/build_gold.py b/eval/build_gold.py index 48c4bd5..e3f85f7 100644 --- a/eval/build_gold.py +++ b/eval/build_gold.py @@ -40,6 +40,9 @@ class simply comes out empty and `run_eval.py` says so instead of pretending. from collections import Counter from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +from sqlite_graph_memory.index_notes import iter_notes # noqa: E402 + WIKILINK_RX = re.compile(r'\[\[([^\]\|#]+)(?:[|#][^\]]*)?\]\]') CYRILLIC_RX = re.compile(r'[а-яё]', re.I) @@ -121,10 +124,10 @@ def read_notes(notes_dir, stems, folders=None): """Every indexed note, parsed into the fields the four classes need.""" root = Path(notes_dir) paths = [] - for p in sorted(root.rglob('*.md')): + # The indexer's own walk: hidden files and dirs, conflict copies and BRAIN_INDEX_HIDDEN + # are decided in one place, so a gold answer is never a note the indexer would skip. + for p in iter_notes(root): rel = p.relative_to(root) - if any(part.startswith('.') for part in rel.parts[:-1]): continue - if 'sync-conflict' in p.name: continue if folders and (not rel.parts[:-1] or rel.parts[0] not in folders): continue if p.stem.lower() not in stems: continue paths.append(p) diff --git a/examples/README.md b/examples/README.md index e2e9ff2..2125d89 100644 --- a/examples/README.md +++ b/examples/README.md @@ -109,7 +109,7 @@ All components share standard configuration variables: | `BRAIN_INDEX_DIR` | `./index` | Directory where `index_notes.py` writes and `brain_ask.py` reads `_brain_e5.npy` and metadata. | | `TURNSTATE_DB` | `./turnstate.db` | SQLite database holding per-turn ledger (`turns`) and A/B telemetry (`ab_recall`). | | `BRAIN_ANSWER_OUT` | `./_brain_answer.txt` | File where the formatted context bundle or answer is mirrored. | -| `BRAIN_INDEX_HIDDEN` | `0` | If `1`, indexes hidden dot-directories too (by default skipped). | +| `BRAIN_INDEX_HIDDEN` | `0` | If `1`, indexes hidden dot-files (`.draft.md`) and dot-directories too (by default both are skipped). | | `BRAIN_AB_JSON` | `None` | If set, `--ab` mode writes a structured JSON diff to this file path. | | `AB_SOURCE` | `work` | Telemetry label written to `ab_recall.source` (`work`, `eval`, etc.). | diff --git a/src/sqlite_graph_memory/index_notes.py b/src/sqlite_graph_memory/index_notes.py index 7d66634..77b2776 100644 --- a/src/sqlite_graph_memory/index_notes.py +++ b/src/sqlite_graph_memory/index_notes.py @@ -12,9 +12,9 @@ python index_notes.py Config (env, optional): BRAIN_INDEX_DIR output dir (default: ./index) - BRAIN_INDEX_HIDDEN 1 = also index dot-directories (default: skip them, - so .stversions/.obsidian/.git and Syncthing conflict - copies never enter the index) + BRAIN_INDEX_HIDDEN 1 = also index dot-files and dot-directories (default: + skip them, so .draft.md, .stversions/.obsidian/.git and + Syncthing conflict copies never enter the index) """ import os, re, sys, pickle from pathlib import Path diff --git a/tests/test_eval.py b/tests/test_eval.py index 14daa80..7c4fe0c 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -158,6 +158,25 @@ def test_a_bridge_question_records_how_many_links_were_cut(tmp_path): assert bridge['gold_truncated'] == 4 +def test_gold_skips_hidden_notes_exactly_like_the_indexer(tmp_path, monkeypatch): + """build_gold walks the vault with the indexer's own filter (issue #25). + + A root-level `.draft.md` is never indexed, so it must never become a question either, + even when a stale index still lists its stem. `BRAIN_INDEX_HIDDEN=1` lets it into the + index, so it must let it into the gold set too: one rule, one place. + """ + vault = _vault(tmp_path) + (vault / '.draft.md').write_text('---\ntitle: "Hidden draft"\n---\n\n' + PARA + '\n', + encoding='utf-8') + stems = INDEXED | {'.draft'} + + monkeypatch.delenv('BRAIN_INDEX_HIDDEN', raising=False) + assert '.draft' not in {n['stem'] for n in read_notes(vault, stems)} + + monkeypatch.setenv('BRAIN_INDEX_HIDDEN', '1') + assert '.draft' in {n['stem'] for n in read_notes(vault, stems)} + + def test_a_filename_living_in_two_folders_is_reported_as_ambiguous(tmp_path): """Found by the review panel: `[[foo]]` names both files, so neither can be gold.""" vault = _vault(tmp_path) diff --git a/tests/test_index_notes_dotfiles.py b/tests/test_index_notes_dotfiles.py index d5d49c5..aad460f 100644 --- a/tests/test_index_notes_dotfiles.py +++ b/tests/test_index_notes_dotfiles.py @@ -35,7 +35,10 @@ def _vault(tmp_path: Path) -> Path: def _names(root: Path) -> set[str]: - return {str(p.relative_to(root)) for p in iter_notes(root)} + # as_posix(): the assertions spell paths with "/". str() gives a backslash on + # Windows, so "sub/.trash.md" would never match and the skip test would pass + # for the wrong reason. + return {p.relative_to(root).as_posix() for p in iter_notes(root)} def test_dot_file_in_root_is_skipped(tmp_path, monkeypatch): @@ -61,3 +64,18 @@ def test_hidden_flag_admits_dot_files_and_dot_dirs(tmp_path, monkeypatch): monkeypatch.setenv("BRAIN_INDEX_HIDDEN", "1") names = _names(_vault(tmp_path)) assert {"ok.md", ".draft.md", "sub/.trash.md", ".drafts/inside.md"} <= names + + +def test_vault_root_that_is_itself_a_dot_directory_is_indexed(tmp_path, monkeypatch): + """Only path components BELOW the vault root decide what is hidden. + + A vault kept at ``~/.vault`` or ``~/.notes`` is ordinary. A filter that checks + the absolute path (``p.parts``) instead of the relative one (``rel.parts``) + sees ``.vault`` in every path and silently returns nothing. + """ + monkeypatch.delenv("BRAIN_INDEX_HIDDEN", raising=False) + root = tmp_path / ".vault" + root.mkdir() + names = _names(_vault(root)) + assert "ok.md" in names + assert ".draft.md" not in names and ".drafts/inside.md" not in names