From e0266956d92e38befe7311fc1a7e1516aa538a98 Mon Sep 17 00:00:00 2001 From: jichao wang Date: Thu, 18 Jun 2026 23:40:38 +0100 Subject: [PATCH 1/2] fix(indexer): treat escaped "\!" gitignore lines as literal, not negation `_normalize_gitignore_lines` unescaped a leading "\#"/"\!" and only *then* checked for negation. For "\!name" the unescape produced "!name", which the negation check misread as a re-include rule, emitting "!**/name". A literal ignore such as `\!important` therefore cancelled an unrelated `important` rule instead of ignoring the file named "!important". Detect the escape before the negation check and skip negation handling for escaped lines, so "\!important" normalizes to "**/!important" (the '!' is no longer pattern-leading, so it is literal). Ordinary negation and escaped "\#" behavior are unchanged. Add tests/test_indexer_gitignore.py covering plain/negated/escaped patterns and an end-to-end GitIgnoreSpec check that the escaped line no longer re-includes unrelated matches. --- src/cocoindex_code/indexer.py | 16 ++++++++--- tests/test_indexer_gitignore.py | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 tests/test_indexer_gitignore.py diff --git a/src/cocoindex_code/indexer.py b/src/cocoindex_code/indexer.py index e028103..63f4e0c 100644 --- a/src/cocoindex_code/indexer.py +++ b/src/cocoindex_code/indexer.py @@ -48,11 +48,19 @@ def _normalize_gitignore_lines(lines: Iterable[str], directory: PurePath) -> lis stripped = line.lstrip() if not stripped or stripped.startswith("#"): continue - if line.startswith("\\#") or line.startswith("\\!"): - line = line[1:] - negated = line.startswith("!") - if negated: + # A leading "\#" or "\!" escapes a literal '#'/'!' — such a line is + # neither a comment nor a negation. Detect the escape *before* the + # negation check, otherwise an escaped "\!foo" is unescaped to "!foo" + # and then misread as a negation that wrongly re-includes unrelated + # "foo" matches. + escaped = line.startswith("\\#") or line.startswith("\\!") + if escaped: line = line[1:] + negated = False + else: + negated = line.startswith("!") + if negated: + line = line[1:] body = line.strip() if not body: continue diff --git a/tests/test_indexer_gitignore.py b/tests/test_indexer_gitignore.py new file mode 100644 index 0000000..e703858 --- /dev/null +++ b/tests/test_indexer_gitignore.py @@ -0,0 +1,50 @@ +"""Unit tests for .gitignore line normalization in the indexer.""" + +from __future__ import annotations + +from pathlib import PurePath + +from pathspec import GitIgnoreSpec + +from cocoindex_code.indexer import _normalize_gitignore_lines + +ROOT = PurePath(".") + + +def test_plain_pattern_is_globbed() -> None: + assert _normalize_gitignore_lines(["build"], ROOT) == ["**/build"] + + +def test_negation_is_preserved() -> None: + assert _normalize_gitignore_lines(["build", "!build/keep.txt"], ROOT) == [ + "**/build", + "!build/keep.txt", + ] + + +def test_escaped_hash_is_literal_not_comment() -> None: + # "\#notacomment" -> a file literally named "#notacomment". + assert _normalize_gitignore_lines(["\\#notacomment"], ROOT) == ["**/#notacomment"] + + +def test_escaped_bang_is_literal_not_negation() -> None: + # Regression: "\!important" means "ignore a file literally named '!important'", + # NOT a negation, so it must not become a "!"-prefixed (negation) pattern. + assert _normalize_gitignore_lines(["\\!important"], ROOT) == ["**/!important"] + + +def test_escaped_bang_does_not_re_include_unrelated_matches() -> None: + # End-to-end: a "\!important" line must not cancel an unrelated "important" + # ignore rule. Before the fix it normalized to "!**/important", which + # re-included every "important" file the previous line had ignored. + spec = GitIgnoreSpec.from_lines( + _normalize_gitignore_lines(["important", "\\!important"], ROOT) + ) + assert spec.match_file("important") is True # still ignored + assert spec.match_file("!important") is True # literal file ignored too + + +def test_subdirectory_prefix_is_applied() -> None: + assert _normalize_gitignore_lines(["\\!keep"], PurePath("sub/dir")) == [ + "sub/dir/**/!keep" + ] From ec2ce38a04dbd2bc224085ba5ccbd04c6ee8e65d Mon Sep 17 00:00:00 2001 From: jichao wang Date: Sun, 21 Jun 2026 15:22:01 +0100 Subject: [PATCH 2/2] fix: keep escape on path-bearing \! / \# gitignore patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addressing review feedback: the previous fix stripped the backslash from an escaped "\!"/"\#" line and relied on the "**/" prefix to keep the leading "!"/"#" from being misread. But a pattern that contains a "/" is anchored and gets no "**/" prefix, so "\!dir/file" normalized to "!dir/file" — which GitIgnoreSpec reads back as a negation (and "\#dir/file" as a comment), dropping the rule. Keep the backslash in the emitted pattern so pathspec parses the "!"/"#" literally in both the prefixed ("**/\!foo") and anchored ("\!dir/file") cases. Adds an end-to-end test for path-bearing escaped patterns; updates the exact-form assertions to the now-escaped output. --- src/cocoindex_code/indexer.py | 9 +++++---- tests/test_indexer_gitignore.py | 22 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/cocoindex_code/indexer.py b/src/cocoindex_code/indexer.py index 63f4e0c..14169f3 100644 --- a/src/cocoindex_code/indexer.py +++ b/src/cocoindex_code/indexer.py @@ -50,12 +50,13 @@ def _normalize_gitignore_lines(lines: Iterable[str], directory: PurePath) -> lis continue # A leading "\#" or "\!" escapes a literal '#'/'!' — such a line is # neither a comment nor a negation. Detect the escape *before* the - # negation check, otherwise an escaped "\!foo" is unescaped to "!foo" - # and then misread as a negation that wrongly re-includes unrelated - # "foo" matches. + # negation check, and KEEP the backslash so the emitted pattern stays + # escaped for pathspec. Stripping it would leave a bare leading "!"/"#" + # — fine when a "**/" prefix is added ("\!foo" -> "**/!foo"), but for a + # path-bearing pattern there is no such prefix ("\!dir/f" -> "!dir/f"), + # and GitIgnoreSpec would then read it back as a negation/comment. escaped = line.startswith("\\#") or line.startswith("\\!") if escaped: - line = line[1:] negated = False else: negated = line.startswith("!") diff --git a/tests/test_indexer_gitignore.py b/tests/test_indexer_gitignore.py index e703858..c749e7f 100644 --- a/tests/test_indexer_gitignore.py +++ b/tests/test_indexer_gitignore.py @@ -23,14 +23,15 @@ def test_negation_is_preserved() -> None: def test_escaped_hash_is_literal_not_comment() -> None: - # "\#notacomment" -> a file literally named "#notacomment". - assert _normalize_gitignore_lines(["\\#notacomment"], ROOT) == ["**/#notacomment"] + # "\#notacomment" -> a file literally named "#notacomment". The escape is + # kept so GitIgnoreSpec does not read the pattern back as a comment. + assert _normalize_gitignore_lines(["\\#notacomment"], ROOT) == ["**/\\#notacomment"] def test_escaped_bang_is_literal_not_negation() -> None: # Regression: "\!important" means "ignore a file literally named '!important'", # NOT a negation, so it must not become a "!"-prefixed (negation) pattern. - assert _normalize_gitignore_lines(["\\!important"], ROOT) == ["**/!important"] + assert _normalize_gitignore_lines(["\\!important"], ROOT) == ["**/\\!important"] def test_escaped_bang_does_not_re_include_unrelated_matches() -> None: @@ -46,5 +47,18 @@ def test_escaped_bang_does_not_re_include_unrelated_matches() -> None: def test_subdirectory_prefix_is_applied() -> None: assert _normalize_gitignore_lines(["\\!keep"], PurePath("sub/dir")) == [ - "sub/dir/**/!keep" + "sub/dir/**/\\!keep" ] + + +def test_escaped_path_bearing_pattern_is_literal() -> None: + # An escaped pattern that contains a "/" is anchored (no "**/" prefix is + # added), so the leading "!"/"#" would sit at the very start of the emitted + # pattern. Keeping the backslash is what stops GitIgnoreSpec from reading it + # back as a negation ("\!dir/file") or a comment ("\#dir/file"). + spec = GitIgnoreSpec.from_lines( + _normalize_gitignore_lines(["\\!dir/file", "\\#dir/other"], ROOT) + ) + assert spec.match_file("dir/file") is False # unescaped sibling, untouched + assert spec.match_file("!dir/file") is True # literal "!dir/file" ignored + assert spec.match_file("#dir/other") is True # literal "#dir/other" ignored