fix(index): skip dot-files, not only dot-directories (closes #25)

CHANGELOG.md

@@ -6,6 +6,14 @@ recorded after the fact rather than backdated to look contemporaneous.
 
 ## Unreleased
 
+**Hidden note files are skipped like hidden folders.** A note whose filename starts with a
+dot, such as `.ideas.md` or `.draft.md`, used to be indexed when it sat in the vault root or a
+visible folder, while the same file inside `.drafts/` was skipped. Now both are skipped, so
+after re-indexing those notes stop showing up in answers. `BRAIN_INDEX_HIDDEN=1` brings back
+the old behaviour and indexes hidden files and folders alike. `eval/build_gold.py` now walks the
+vault with the indexer's own filter, so it follows the same rule and the same variable instead
+of keeping its own copy (issue #25).
+
 **The PageRank lane the README credits is now in the package** (#51). `ppr.py` runs
 personalized PageRank over the `[[wikilink]]` graph, seeded by the dense hits, and fuses it with
 the reranker by RRF while the reranker's top 5 stay where they are. Off by default (`SGM_PPR=1`

eval/build_gold.py

@@ -40,6 +40,9 @@ class simply comes out empty and `run_eval.py` says so instead of pretending.
 from collections import Counter
 from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src'))
+from sqlite_graph_memory.index_notes import iter_notes  # noqa: E402
+
 WIKILINK_RX = re.compile(r'\[\[([^\]\|#]+)(?:[|#][^\]]*)?\]\]')
 CYRILLIC_RX = re.compile(r'[а-яё]', re.I)
 
@@ -121,10 +124,10 @@ def read_notes(notes_dir, stems, folders=None):
     """Every indexed note, parsed into the fields the four classes need."""
     root = Path(notes_dir)
     paths = []
-    for p in sorted(root.rglob('*.md')):
+    # The indexer's own walk: hidden files and dirs, conflict copies and BRAIN_INDEX_HIDDEN
+    # are decided in one place, so a gold answer is never a note the indexer would skip.
+    for p in iter_notes(root):
         rel = p.relative_to(root)
-        if any(part.startswith('.') for part in rel.parts[:-1]): continue
-        if 'sync-conflict' in p.name: continue
         if folders and (not rel.parts[:-1] or rel.parts[0] not in folders): continue
         if p.stem.lower() not in stems: continue
         paths.append(p)

examples/README.md

@@ -109,7 +109,7 @@ All components share standard configuration variables:
 | `BRAIN_INDEX_DIR` | `./index` | Directory where `index_notes.py` writes and `brain_ask.py` reads `_brain_e5.npy` and metadata. |
 | `TURNSTATE_DB` | `./turnstate.db` | SQLite database holding per-turn ledger (`turns`) and A/B telemetry (`ab_recall`). |
 | `BRAIN_ANSWER_OUT` | `./_brain_answer.txt` | File where the formatted context bundle or answer is mirrored. |
-| `BRAIN_INDEX_HIDDEN` | `0` | If `1`, indexes hidden dot-directories too (by default skipped). |
+| `BRAIN_INDEX_HIDDEN` | `0` | If `1`, indexes hidden dot-files (`.draft.md`) and dot-directories too (by default both are skipped). |
 | `BRAIN_AB_JSON` | `None` | If set, `--ab` mode writes a structured JSON diff to this file path. |
 | `AB_SOURCE` | `work` | Telemetry label written to `ab_recall.source` (`work`, `eval`, etc.). |
 

src/sqlite_graph_memory/index_notes.py

@@ -12,9 +12,9 @@
   python index_notes.py <notes_dir>
 Config (env, optional):
   BRAIN_INDEX_DIR     output dir (default: ./index)
-  BRAIN_INDEX_HIDDEN  1 = also index dot-directories (default: skip them,
-                      so .stversions/.obsidian/.git and Syncthing conflict
-                      copies never enter the index)
+  BRAIN_INDEX_HIDDEN  1 = also index dot-files and dot-directories (default:
+                      skip them, so .draft.md, .stversions/.obsidian/.git and
+                      Syncthing conflict copies never enter the index)
 """
 import os, re, sys, pickle
 from pathlib import Path
@@ -77,15 +77,15 @@ def fm_date(fm):
 def iter_notes(notes_dir):
     """Files that count as a note under ``notes_dir``.
 
-    Skips dot-directories and Syncthing conflict copies. Set ``BRAIN_INDEX_HIDDEN=1``
-    to index hidden folders too — someone will legitimately keep notes in one.
+    Skips dot-directories, dot-files (``.draft.md``) and Syncthing conflict copies.
+    Set ``BRAIN_INDEX_HIDDEN=1`` to index hidden paths too — someone will legitimately keep notes in one.
     """
     root = Path(notes_dir)
     include_hidden = os.getenv('BRAIN_INDEX_HIDDEN', '') not in ('', '0', 'false', 'no')
     out = []
     for p in root.rglob('*.md'):
         rel = p.relative_to(root)
-        if not include_hidden and any(part.startswith('.') for part in rel.parts[:-1]):
+        if not include_hidden and any(part.startswith('.') for part in rel.parts):
             continue
         if CONFLICT_MARKER in p.name:
             continue

tests/test_eval.py

@@ -158,6 +158,25 @@ def test_a_bridge_question_records_how_many_links_were_cut(tmp_path):
     assert bridge['gold_truncated'] == 4
 
 
+def test_gold_skips_hidden_notes_exactly_like_the_indexer(tmp_path, monkeypatch):
+    """build_gold walks the vault with the indexer's own filter (issue #25).
+
+    A root-level `.draft.md` is never indexed, so it must never become a question either,
+    even when a stale index still lists its stem. `BRAIN_INDEX_HIDDEN=1` lets it into the
+    index, so it must let it into the gold set too: one rule, one place.
+    """
+    vault = _vault(tmp_path)
+    (vault / '.draft.md').write_text('---\ntitle: "Hidden draft"\n---\n\n' + PARA + '\n',
+                                     encoding='utf-8')
+    stems = INDEXED | {'.draft'}
+
+    monkeypatch.delenv('BRAIN_INDEX_HIDDEN', raising=False)
+    assert '.draft' not in {n['stem'] for n in read_notes(vault, stems)}
+
+    monkeypatch.setenv('BRAIN_INDEX_HIDDEN', '1')
+    assert '.draft' in {n['stem'] for n in read_notes(vault, stems)}
+
+
 def test_a_filename_living_in_two_folders_is_reported_as_ambiguous(tmp_path):
     """Found by the review panel: `[[foo]]` names both files, so neither can be gold."""
     vault = _vault(tmp_path)

tests/test_index_notes_dotfiles.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+"""Dot-FILES are skipped the same way dot-DIRECTORIES are (issue #25).
+
+`iter_notes` filtered hidden *directories* by checking `rel.parts[:-1]` — every
+path component except the last. The last component is the filename, so a note
+called `.draft.md` or `.trash.md` sitting in the vault root (or in any visible
+folder) was indexed as if it were a real note, while the same file one level
+down inside `.drafts/` was correctly skipped. Reproduced 2026-10-13 with
+`.secret.md` next to `ok.md`: `iter_notes()` returned both.
+
+RED-FIRST: restore `rel.parts[:-1]` on the filter line and
+`test_dot_file_in_root_is_skipped` fails with `.draft.md` in the result.
+
+Both halves are pinned on purpose — the directory case already worked and must
+keep working, the file case is the fix — and `BRAIN_INDEX_HIDDEN=1` must still
+let both through, because someone legitimately keeps notes in a hidden folder.
+
+No model download, no network: pure file work.
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+from sqlite_graph_memory.index_notes import iter_notes  # noqa: E402
+
+
+def _vault(tmp_path: Path) -> Path:
+    (tmp_path / "ok.md").write_text("visible note", encoding="utf-8")
+    (tmp_path / ".draft.md").write_text("hidden file in root", encoding="utf-8")
+    (tmp_path / "sub").mkdir()
+    (tmp_path / "sub" / ".trash.md").write_text("hidden file in visible dir", encoding="utf-8")
+    (tmp_path / ".drafts").mkdir()
+    (tmp_path / ".drafts" / "inside.md").write_text("visible name, hidden dir", encoding="utf-8")
+    return tmp_path
+
+
+def _names(root: Path) -> set[str]:
+    # as_posix(): the assertions spell paths with "/". str() gives a backslash on
+    # Windows, so "sub/.trash.md" would never match and the skip test would pass
+    # for the wrong reason.
+    return {p.relative_to(root).as_posix() for p in iter_notes(root)}
+
+
+def test_dot_file_in_root_is_skipped(tmp_path, monkeypatch):
+    monkeypatch.delenv("BRAIN_INDEX_HIDDEN", raising=False)
+    assert ".draft.md" not in _names(_vault(tmp_path))
+
+
+def test_dot_file_in_visible_dir_is_skipped(tmp_path, monkeypatch):
+    monkeypatch.delenv("BRAIN_INDEX_HIDDEN", raising=False)
+    assert "sub/.trash.md" not in _names(_vault(tmp_path))
+
+
+def test_dot_dir_still_skipped_and_visible_note_still_kept(tmp_path, monkeypatch):
+    """The pre-existing behaviour must survive the fix."""
+    monkeypatch.delenv("BRAIN_INDEX_HIDDEN", raising=False)
+    names = _names(_vault(tmp_path))
+    assert ".drafts/inside.md" not in names
+    assert "ok.md" in names
+
+
+def test_hidden_flag_admits_dot_files_and_dot_dirs(tmp_path, monkeypatch):
+    """Opt-in must cover both kinds of hidden path, not just directories."""
+    monkeypatch.setenv("BRAIN_INDEX_HIDDEN", "1")
+    names = _names(_vault(tmp_path))
+    assert {"ok.md", ".draft.md", "sub/.trash.md", ".drafts/inside.md"} <= names
+
+
+def test_vault_root_that_is_itself_a_dot_directory_is_indexed(tmp_path, monkeypatch):
+    """Only path components BELOW the vault root decide what is hidden.
+
+    A vault kept at ``~/.vault`` or ``~/.notes`` is ordinary. A filter that checks
+    the absolute path (``p.parts``) instead of the relative one (``rel.parts``)
+    sees ``.vault`` in every path and silently returns nothing.
+    """
+    monkeypatch.delenv("BRAIN_INDEX_HIDDEN", raising=False)
+    root = tmp_path / ".vault"
+    root.mkdir()
+    names = _names(_vault(root))
+    assert "ok.md" in names
+    assert ".draft.md" not in names and ".drafts/inside.md" not in names