diff --git a/openkb/lint.py b/openkb/lint.py index c3873afd..8f08d8d7 100644 --- a/openkb/lint.py +++ b/openkb/lint.py @@ -20,8 +20,33 @@ from openkb.locks import atomic_write_text from openkb.schema import PAGE_CONTENT_DIRS -# Matches [[wikilink]] or [[subdir/link]] -_WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]") +# Matches [[target]], [[target|alias]], [[target#Heading]], [[target#^block]], +# same-page fragment links [[#Heading]], and embeds ![[file]]. Named groups +# keep the pieces separable so Obsidian fragment/embed syntax survives +# validation and rewriting instead of being demoted to plain text. +_WIKILINK_RE = re.compile( + r"(?P!)?\[\[(?P[^\]|#]*)(?P#[^\]|]*)?(?:\|(?P[^\]]+))?\]\]" +) + +# Extension-like suffix on a wikilink target: 1-8 alphanumeric chars with at +# least one letter (".txt", ".flac", ".drawio", ".md" — but not the ".11420" +# of a dotted page name like "2509.11420"). Obsidian accepts arbitrary +# attachment formats, so no closed extension list can be complete; targets +# that look like files are simply never demoted. Keeping a genuinely dead +# link is visible and harmless — demoting a valid attachment embed to plain +# text is silent data loss. +_EXTENSION_RE = re.compile(r"\.(?=[A-Za-z0-9]{0,7}[A-Za-z])[A-Za-z0-9]{1,8}$") + + +def _is_attachment(target: str) -> bool: + """True when a wikilink target looks like a file attachment, not a page. + + Checked only *after* page-target validation fails, so a real page whose + name happens to carry an extension-like suffix (``concepts/node.js``) + still validates and rewrites normally. + """ + return bool(_EXTENSION_RE.search(target.rsplit("/", 1)[-1])) + # Files to exclude from lint scanning (schema, logs, etc.) _EXCLUDED_FILES = {"AGENTS.md", "SCHEMA.md", "log.md"} @@ -68,15 +93,28 @@ def strip_ghost_wikilinks( ) -> tuple[str, list[str]]: """Strip [[wikilinks]] whose targets do not exist in ``known_targets``. - For each ``[[X]]`` or ``[[X|alias]]`` in ``content``: + For each ``[[X]]``, ``[[X|alias]]``, ``[[X#frag]]``, or + ``[[X#frag|alias]]`` in ``content``: - If ``X`` is in ``known_targets`` exactly, the link is kept as-is. - Otherwise, ``X`` is normalized (see :func:`_normalize_target`) and matched against the normalized form of each known target. On a hit, - the link is rewritten to the canonical target form. + the link is rewritten to the canonical target form, preserving any + ``#Heading`` / ``#^block`` fragment and alias. - Otherwise, the brackets are removed and the link becomes plain text (the alias if provided, otherwise the slug rendered as words). + Obsidian syntax that never targets a wiki page is passed through + untouched: same-page fragment links (``[[#Heading]]``) and attachment + embeds/links — any unknown target with an extension-like suffix + (``![[file.png]]``, ``[[report.pdf]]``, ``![[audio.flac]]``, + ``![[diagram.drawio]]``). Obsidian accepts arbitrary attachment + formats, so classification is by shape, not by a closed extension + list; page validation runs first, so a real page named like a file + (``concepts/node.js``) still validates and rewrites normally. Note + embeds (``![[concepts/attention]]``) target pages like any other + link and go through the same validation, keeping the ``!`` on rewrite. + Args: content: Markdown text containing zero or more ``[[wikilinks]]``. known_targets: Valid link targets, e.g. @@ -96,25 +134,33 @@ def strip_ghost_wikilinks( ghosts: list[str] = [] def _repl(m: re.Match) -> str: - raw = m.group(1) - if "|" in raw: - target, alias = raw.split("|", 1) - target = target.strip() - alias = alias.strip() - else: - target = raw.strip() - alias = None - - # Direct hit + embed = m.group("embed") or "" + target = (m.group("target") or "").strip() + frag = m.group("frag") or "" + alias = (m.group("alias") or "").strip() or None + + # [[#Heading]] links within the same page carry no target to check. + if not target: + return m.group(0) + + # Direct hit — page targets validate first, so a page whose name + # carries an extension-like suffix still resolves normally. if target in known_targets: return m.group(0) - # Fuzzy normalized hit → rewrite to canonical + # Fuzzy normalized hit → rewrite to canonical, keeping the + # fragment and the embed marker canonical = norm_index.get(_normalize_target(target)) if canonical is not None: if alias: - return f"[[{canonical}|{alias}]]" - return f"[[{canonical}]]" + return f"{embed}[[{canonical}{frag}|{alias}]]" + return f"{embed}[[{canonical}{frag}]]" + + # Unknown target that looks like a file (![[audio.flac]], + # [[report.pdf]], ![[diagram.drawio]]): an attachment, not a page — + # keep verbatim rather than risk demoting a valid embed. + if _is_attachment(target): + return m.group(0) # Ghost — strip brackets, leave readable display ghosts.append(target) @@ -152,12 +198,23 @@ def _all_wiki_pages(wiki: Path) -> dict[str, Path]: def _extract_wikilinks(text: str) -> list[str]: - """Return all wikilink targets found in *text*. - - Handles ``[[target|display text]]`` alias syntax — only the target is returned. + """Return all page-link targets found in *text*. + + Aliases and ``#Heading`` / ``#^block`` fragments are dropped — only the + page part is returned. Note embeds (``![[concepts/x]]``) count as page + links; attachment embeds/links (any extension-like target — + ``![[fig.png]]``, ``[[report.pdf]]``, ``![[audio.flac]]``) and + same-page fragment links (``[[#Heading]]``) are skipped — they never + target a wiki page, so reporting them as broken would be a false + positive. """ - raw = _WIKILINK_RE.findall(text) - return [link.split("|")[0].strip() for link in raw] + targets: list[str] = [] + for m in _WIKILINK_RE.finditer(text): + target = (m.group("target") or "").strip() + if not target or _is_attachment(target): + continue + targets.append(target) + return targets def list_existing_wiki_targets(wiki_dir: Path) -> set[str]: diff --git a/tests/test_lint.py b/tests/test_lint.py index ccc2d53b..e9e3e4d2 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -545,6 +545,223 @@ def test_accepts_prebuilt_norm_index_with_identical_result(self): assert "[[concepts/missing]]" not in out_b +class TestObsidianSyntax: + """Obsidian link syntax beyond ``[[page]]`` / ``[[page|alias]]``. + + Heading links (``[[page#H]]``), block refs (``[[page#^b]]``), same-page + fragments (``[[#H]]``), embeds (``![[file.png]]``), and attachment + links (``[[file.pdf]]``) are all valid in Obsidian. The linter must + neither demote them to plain text (strip/fix) nor report them as + broken (find_broken_links). + """ + + # --- strip_ghost_wikilinks ------------------------------------------- + + def test_keeps_heading_fragment_on_direct_match(self): + out, ghosts = strip_ghost_wikilinks( + "See [[concepts/attention#Scaled Dot-Product]].", + {"concepts/attention"}, + ) + assert out == "See [[concepts/attention#Scaled Dot-Product]]." + assert ghosts == [] + + def test_keeps_block_ref_on_direct_match(self): + out, ghosts = strip_ghost_wikilinks( + "See [[concepts/attention#^abc123]].", + {"concepts/attention"}, + ) + assert out == "See [[concepts/attention#^abc123]]." + assert ghosts == [] + + def test_keeps_fragment_with_alias(self): + out, ghosts = strip_ghost_wikilinks( + "See [[concepts/attention#Scores|the scores]].", + {"concepts/attention"}, + ) + assert out == "See [[concepts/attention#Scores|the scores]]." + assert ghosts == [] + + def test_preserves_fragment_through_fuzzy_rewrite(self): + out, ghosts = strip_ghost_wikilinks( + "See [[concepts/gist_memory#Notes]].", + {"concepts/gist-memory"}, + ) + assert out == "See [[concepts/gist-memory#Notes]]." + assert ghosts == [] + + def test_preserves_fragment_and_alias_through_fuzzy_rewrite(self): + out, ghosts = strip_ghost_wikilinks( + "See [[concepts/gist_memory#Notes|notes]].", + {"concepts/gist-memory"}, + ) + assert out == "See [[concepts/gist-memory#Notes|notes]]." + assert ghosts == [] + + def test_ghost_with_fragment_demoted_without_fragment_text(self): + out, ghosts = strip_ghost_wikilinks( + "See [[concepts/missing#Section]].", + {"concepts/attention"}, + ) + assert out == "See missing." + assert ghosts == ["concepts/missing"] + + def test_uncommon_attachment_extensions_left_untouched(self): + # Obsidian accepts arbitrary attachment formats — classification is + # by extension *shape*, not a closed list (the P1 regression: these + # were silently demoted to plain text). + text = "![[notes.txt]] ![[audio.flac]] ![[diagram.drawio]] [[note.md]]" + out, ghosts = strip_ghost_wikilinks(text, set()) + assert out == text + assert ghosts == [] + + def test_dotted_page_name_still_ghosted(self): + # A numeric dotted suffix is not extension-like — an unknown page + # target such as an unsanitized arXiv id still gets demoted. + out, ghosts = strip_ghost_wikilinks( + "See [[summaries/2509.11420]].", + {"summaries/other"}, + ) + assert out == "See 2509.11420." + assert ghosts == ["summaries/2509.11420"] + + def test_page_named_like_file_validates_before_attachment_check(self): + # Page validation runs first: a real page whose name carries an + # extension-like suffix still direct-matches and fuzzy-rewrites. + out, ghosts = strip_ghost_wikilinks( + "See [[concepts/node.js]] and [[concepts/Node.js]].", + {"concepts/node.js"}, + ) + assert out == "See [[concepts/node.js]] and [[concepts/node.js]]." + assert ghosts == [] + + def test_attachment_embed_left_untouched(self): + text = "Figure: ![[images/doc/p1_img1.png]] here." + out, ghosts = strip_ghost_wikilinks(text, set()) + assert out == text + assert ghosts == [] + + def test_note_embed_kept_on_direct_match(self): + # ![[page]] embeds a note — a page link like any other, keep the "!". + out, ghosts = strip_ghost_wikilinks( + "Embedded: ![[concepts/attention]].", + {"concepts/attention"}, + ) + assert out == "Embedded: ![[concepts/attention]]." + assert ghosts == [] + + def test_note_embed_fuzzy_rewrite_preserves_embed_marker(self): + out, ghosts = strip_ghost_wikilinks( + "Embedded: ![[concepts/Gist_Memory#Notes]].", + {"concepts/gist-memory"}, + ) + assert out == "Embedded: ![[concepts/gist-memory#Notes]]." + assert ghosts == [] + + def test_ghost_note_embed_demoted_without_bang_residue(self): + out, ghosts = strip_ghost_wikilinks( + "Embedded: ![[concepts/missing]].", + {"concepts/attention"}, + ) + assert out == "Embedded: missing." + assert ghosts == ["concepts/missing"] + + def test_attachment_link_left_untouched(self): + text = "Original: [[report.pdf]]." + out, ghosts = strip_ghost_wikilinks(text, set()) + assert out == text + assert ghosts == [] + + def test_same_page_fragment_left_untouched(self): + text = "Jump to [[#Conclusiones]]." + out, ghosts = strip_ghost_wikilinks(text, set()) + assert out == text + assert ghosts == [] + + # --- find_broken_links ------------------------------------------------ + + def test_heading_link_to_existing_page_not_broken(self, tmp_path): + wiki = _make_wiki(tmp_path) + (wiki / "concepts" / "attention.md").write_text("# Attention", encoding="utf-8") + (wiki / "summaries" / "paper.md").write_text( + "See [[concepts/attention#Scores]] and [[concepts/attention#^b1]].", + encoding="utf-8", + ) + + assert find_broken_links(wiki) == [] + + def test_embed_and_attachment_links_not_reported_broken(self, tmp_path): + wiki = _make_wiki(tmp_path) + (wiki / "summaries" / "paper.md").write_text( + "![[images/doc/fig.png]] and [[scan.pdf]] and [[#Local]] " + "and ![[audio.flac]] and ![[diagram.drawio]] and [[note.md]].", + encoding="utf-8", + ) + + assert find_broken_links(wiki) == [] + + def test_heading_link_to_missing_page_still_broken(self, tmp_path): + wiki = _make_wiki(tmp_path) + (wiki / "summaries" / "paper.md").write_text( + "See [[concepts/ghost#Section]].", encoding="utf-8" + ) + + result = find_broken_links(wiki) + + assert len(result) == 1 + assert "ghost" in result[0] + + def test_note_embed_validated_as_page_link(self, tmp_path): + # ![[page]] note embeds participate in lint: a broken one is + # reported, an existing one is not. + wiki = _make_wiki(tmp_path) + (wiki / "concepts" / "attention.md").write_text("# Attention", encoding="utf-8") + (wiki / "summaries" / "paper.md").write_text( + "![[concepts/attention]] and ![[concepts/ghost]].", encoding="utf-8" + ) + + result = find_broken_links(wiki) + + assert len(result) == 1 + assert "ghost" in result[0] + + def test_note_embed_counts_as_incoming_link_for_orphans(self, tmp_path): + # A page referenced only via ![[embed]] is linked, not an orphan. + wiki = _make_wiki(tmp_path) + (wiki / "concepts" / "embedded.md").write_text("# Embedded", encoding="utf-8") + (wiki / "summaries" / "host.md").write_text("![[concepts/embedded]]", encoding="utf-8") + + result = find_orphans(wiki) + + assert "concepts/embedded" not in result + + # --- fix_broken_links regression on explorations/ --------------------- + + def test_fix_is_idempotent_on_handwritten_exploration(self, tmp_path): + """A manually written note in explorations/ using the full Obsidian + syntax must survive a wiki-wide ``lint --fix`` sweep unchanged — + this was the data-loss path: heading/block/embed links were being + demoted to plain text.""" + wiki = _make_wiki(tmp_path) + (wiki / "concepts" / "attention.md").write_text("# Attention", encoding="utf-8") + (wiki / "explorations").mkdir() + note = wiki / "explorations" / "notas-manuales.md" + original = ( + "# Notas\n\n" + "Ver [[concepts/attention#Scaled Dot-Product]] y " + "[[concepts/attention#^bloque1|el bloque]].\n\n" + "![[images/doc/p1_img1.png]]\n\n" + "Nota embebida: ![[concepts/attention]]\n\n" + "Adjunto: [[informe.pdf]] — y volver a [[#Notas]].\n" + ) + note.write_text(original, encoding="utf-8") + + files_changed, ghosts = fix_broken_links(wiki) + + assert note.read_text(encoding="utf-8") == original + assert files_changed == 0 + assert ghosts == 0 + + class TestBuildNormIndex: def test_returns_normalized_to_canonical_map(self): idx = build_norm_index({"concepts/Gist_Memory", "summaries/Paper"})