From 92715035ba0a8f824901db452390a1fd03ccf218 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 1 Nov 2026 01:08:04 +0100 Subject: [PATCH 1/4] refactor(source): share the line index of JSON and TOML positions The line index that places the values of JSON and TOML documents finds the start of each line once, then the line of an offset by bisection. It moves to source.manager, beside LINE_BREAK and lines_of, whose count of lines it follows, and is exported as LineIndex, so that a template's positions can be found with it. --- src/coretrace_python/source/__init__.py | 9 ++++++++- src/coretrace_python/source/manager.py | 21 +++++++++++++++++++++ src/coretrace_python/source/positions.py | 23 +++-------------------- tests/test_line_index.py | 23 +++++++++++++++++++++++ 4 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 tests/test_line_index.py diff --git a/src/coretrace_python/source/__init__.py b/src/coretrace_python/source/__init__.py index aa2756b..cb36428 100644 --- a/src/coretrace_python/source/__init__.py +++ b/src/coretrace_python/source/__init__.py @@ -1,6 +1,12 @@ """Source storage and location primitives.""" -from coretrace_python.source.manager import LINE_BREAK, SourceManager, decode_text, lines_of +from coretrace_python.source.manager import ( + LINE_BREAK, + LineIndex, + SourceManager, + decode_text, + lines_of, +) from coretrace_python.source.model import ( FileLocation, ImportName, @@ -14,6 +20,7 @@ "LINE_BREAK", "FileLocation", "ImportName", + "LineIndex", "Location", "SourceFile", "SourceId", diff --git a/src/coretrace_python/source/manager.py b/src/coretrace_python/source/manager.py index 248c2f9..ee48afe 100644 --- a/src/coretrace_python/source/manager.py +++ b/src/coretrace_python/source/manager.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from bisect import bisect_right from pathlib import Path from coretrace_python.source.model import ImportName, SourceFile, SourceId @@ -86,6 +87,26 @@ def lines_of(text: str) -> list[str]: return lines +class LineIndex: + """The line and column of each offset of a text, lines counted as ``lines_of`` + counts them. The start of each line is found once; the line of an offset is then + found by bisection, in time logarithmic in the lines.""" + + def __init__(self, text: str) -> None: + self._starts = [0, *(match.end() for match in LINE_BREAK.finditer(text))] + + def position(self, offset: int) -> tuple[int, int]: + """The one-based line and column of ``offset``.""" + + line = bisect_right(self._starts, offset) + return line, offset - self._starts[line - 1] + 1 + + def offset(self, line: int, column: int) -> int: + """The offset of the one-based ``line`` and ``column``.""" + + return self._starts[line - 1] + column - 1 + + class SourceManager: """Own source text and return one canonical object for each source ID.""" diff --git a/src/coretrace_python/source/positions.py b/src/coretrace_python/source/positions.py index f5f8dc9..ae4e173 100644 --- a/src/coretrace_python/source/positions.py +++ b/src/coretrace_python/source/positions.py @@ -16,9 +16,8 @@ import json import re import tomllib -from bisect import bisect_right -from coretrace_python.source.manager import LINE_BREAK, lines_of +from coretrace_python.source.manager import LineIndex, lines_of from coretrace_python.source.model import FileLocation, SourceId Pointer = tuple[str | int, ...] @@ -54,22 +53,6 @@ def pointer_text(path: Pointer) -> str: return "".join("/" + str(part).replace("~", "~0").replace("/", "~1") for part in path) -class _Lines: - def __init__(self, text: str) -> None: - self._starts = [0, *(match.end() for match in LINE_BREAK.finditer(text))] - - def position(self, offset: int) -> tuple[int, int]: - """The one-based line and column of ``offset``.""" - - line = bisect_right(self._starts, offset) - return line, offset - self._starts[line - 1] + 1 - - def offset(self, line: int, column: int) -> int: - """The offset of the one-based ``line`` and ``column``.""" - - return self._starts[line - 1] + column - 1 - - def _same(found: object, value: object) -> bool: return type(found) is type(value) and found == value @@ -87,7 +70,7 @@ def __init__(self, text: str) -> None: raise ValueError("trailing text") except (ValueError, IndexError, RecursionError): self._values = {} - self._lines = _Lines(text) + self._lines = LineIndex(text) def locate(self, path: Pointer, value: object) -> tuple[int, int] | None: """The line and column ``value`` starts at under ``path``, or None when the text @@ -183,7 +166,7 @@ def __init__(self, text: str) -> None: self._lines = lines_of(text) # The text with one line feed per line break, where arrays are read, once each. self._text = "\n".join(self._lines) - self._offsets = _Lines(self._text) + self._offsets = LineIndex(self._text) self._arrays: dict[int, list[tuple[int, int]]] = {} state = _LexState() for number, line in enumerate(self._lines, start=1): diff --git a/tests/test_line_index.py b/tests/test_line_index.py new file mode 100644 index 0000000..980fb52 --- /dev/null +++ b/tests/test_line_index.py @@ -0,0 +1,23 @@ +"""The line index of a text: the line and column of each offset, lines counted as +``lines_of`` counts them, and back.""" + +from __future__ import annotations + +from coretrace_python.source import LineIndex + +# Lines broken by a line feed, a carriage return and line feed, and a lone carriage +# return; a column counts characters, not bytes. +TEXT = "a\nbé\r\nc\rd" +POSITIONS = [(1, 1), (1, 2), (2, 1), (2, 2), (2, 3), (2, 4), (3, 1), (3, 2), (4, 1), (4, 2)] + + +def test_each_offset_has_the_line_and_column_lines_of_counts() -> None: + index = LineIndex(TEXT) + + assert [index.position(offset) for offset in range(len(TEXT) + 1)] == POSITIONS + + +def test_each_position_has_its_offset_back() -> None: + index = LineIndex(TEXT) + + assert [index.offset(line, column) for line, column in POSITIONS] == list(range(len(TEXT) + 1)) From c16c595ab951287e03deb614fd0ebd7cafe5898e Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 1 Nov 2026 01:08:04 +0100 Subject: [PATCH 2/4] refactor(templates): give each include, extends and load of a template its span A template's includes, extends and loads are read with the span they stand at, as its filters are, instead of their offset in its text. The unread entries take their line from that span, without the text. The positions are still counted by _position. --- src/coretrace_python/taint/templates.py | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index 71b44c0..d097dfb 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -935,13 +935,13 @@ class _Application: @dataclass(frozen=True) class _Reference: - """A template another one includes, or ``extends``, at ``offset`` in its ``blocks``: + """A template another one includes, or ``extends``, at ``span`` in its ``blocks``: by ``name``, None when an expression names it. The referenced template's free names are ``bindings``, then, unless ``only``, what the ``scope`` binds there, then the referencing template's.""" name: str | None - offset: int + span: SourceSpan bindings: Mapping[str, frozenset[str]] = field(default_factory=dict) only: bool = False scope: Mapping[str, frozenset[str]] = field(default_factory=dict) @@ -961,12 +961,12 @@ def maps(self, names: frozenset[str]) -> frozenset[str]: @dataclass(frozen=True) class _Load: - """A ``{% load %}`` of ``library`` at ``offset``, of the ``names`` it selects + """A ``{% load %}`` of ``library`` at ``span``, of the ``names`` it selects (``{% load a b from lib %}``), None for the whole library.""" library: str names: frozenset[str] | None - offset: int + span: SourceSpan @dataclass(frozen=True) @@ -1027,7 +1027,7 @@ def project_templates(root: Path) -> ProjectTemplates: continue read[path] = _read_scopes(text, SourceId(str(path)), libraries) for reference in read[path].references: - where = f"{relative}:{_position(text, reference.offset)[0]}" + where = f"{relative}:{reference.span.start_line}" if reference.name is None: unread.append(f"{where} names a template by an expression") elif reference.name not in files: @@ -1035,7 +1035,7 @@ def project_templates(root: Path) -> ProjectTemplates: for load in read[path].unresolved: library = libraries.get(load.library) reason = "not found" if library is None else "not read" if library.uncertain is None else "found in several files" - unread.append(f"{relative}:{_position(text, load.offset)[0]} loads {load.library!r}, {reason}") + unread.append(f"{relative}:{load.span.start_line} loads {load.library!r}, {reason}") calls = tuple(FilterCall(a.symbol, a.span) for template in read.values() for a in template.applications) # What each template reaches rendered on its own, for this analysis only. settled = _settled(files, read) @@ -1316,6 +1316,10 @@ def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library]) loaded: list[_Load] = [] unresolved: dict[_Load, None] = {} + def span(offset: int) -> SourceSpan: + line, column = _position(text, offset) + return SourceSpan(source, line, column) + def free(names: frozenset[str]) -> frozenset[str]: found: set[str] = set() for name in names: @@ -1354,9 +1358,7 @@ def apply(head: frozenset[str], chain: list[tuple[str, int, frozenset[str]]]) -> if isinstance(symbol, _Load): unresolved[symbol] = None elif symbol is not None: - line, column = _position(text, offset) - span = SourceSpan(source, line, column) - applications.append(_Application(symbol, span, value, free(argument), tuple(opened))) + applications.append(_Application(symbol, span(offset), value, free(argument), tuple(opened))) value |= free(argument) return value @@ -1414,13 +1416,13 @@ def apply(head: frozenset[str], chain: list[tuple[str, int, frozenset[str]]]) -> elif name in ("include", "extends"): named = _constant(words[1]) if len(words) > 1 else None if name == "extends": - references.append(_Reference(named, offset, extends=True)) + references.append(_Reference(named, span(offset), extends=True)) else: context = {k: v for scope in scopes for k, v in scope.items()} - references.append(_Reference(named, offset, bindings, "only" in words, context, tuple(opened))) + references.append(_Reference(named, span(offset), bindings, "only" in words, context, tuple(opened))) elif name == "load": selected = frozenset(words[1 : words.index("from")]) if "from" in words else None - loaded.extend(_Load(library, selected, offset) for library in _loaded(words[1:])) + loaded.extend(_Load(library, selected, span(offset)) for library in _loaded(words[1:])) elif "as" in words[1:-1]: target = words[words.index("as", 1) + 1] scopes[-1][target] = values.get(words[1], frozenset()) if name == "regroup" else frozenset() From bf0bac352bc41c1f3047661b39449804cae685e1 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 1 Nov 2026 01:08:04 +0100 Subject: [PATCH 3/4] fix(templates): find the positions in a template from a line index built once Each position in a template was counted from its start: collecting the filters of a template applying n of them read it n times, in time quadratic in its size (#288). A template of 2 MB applying 160,000 filters on one line took 49 s, where Django compiles and renders it in 2.7 s. The line of each include and extends, and of each load the engine cannot resolve, was counted the same way. The start of each line of a template is now found once, when it is read, and the line of each position by bisection. The positions do not change: templates are read with universal newlines, so their text holds only line feeds, which both count alike. --- src/coretrace_python/taint/templates.py | 11 ++-- tests/test_template_positions.py | 68 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 tests/test_template_positions.py diff --git a/src/coretrace_python/taint/templates.py b/src/coretrace_python/taint/templates.py index d097dfb..39f6dae 100644 --- a/src/coretrace_python/taint/templates.py +++ b/src/coretrace_python/taint/templates.py @@ -79,7 +79,7 @@ analyze_scopes, ) from coretrace_python.semantic.symbols import SymbolId, analyze_symbols -from coretrace_python.source import SourceId, SourceManager, SourceSpan +from coretrace_python.source import LineIndex, SourceId, SourceManager, SourceSpan from coretrace_python.taint.models import ModelTable TEMPLATES_DIRECTORY = "templates" @@ -1315,9 +1315,10 @@ def _read_scopes(text: str, source: SourceId, libraries: Mapping[str, _Library]) blocks: list[tuple[list[tuple[str, int, frozenset[str]]], set[str]]] = [] loaded: list[_Load] = [] unresolved: dict[_Load, None] = {} + lines = LineIndex(text) def span(offset: int) -> SourceSpan: - line, column = _position(text, offset) + line, column = lines.position(offset) return SourceSpan(source, line, column) def free(names: frozenset[str]) -> frozenset[str]: @@ -1706,12 +1707,6 @@ def _expressions(text: str) -> Iterator[tuple[int, str, bool]]: yield match.start(2), tag, True -def _position(text: str, offset: int) -> tuple[int, int]: - """The one-based line and column of ``offset`` in ``text``.""" - - return text.count("\n", 0, offset) + 1, offset - text.rfind("\n", 0, offset) - - class EscapedTemplates(Analysis[frozenset[str]]): """The names of the project templates that escape everything they render, provided by the engine for a project; none on its own.""" diff --git a/tests/test_template_positions.py b/tests/test_template_positions.py new file mode 100644 index 0000000..07b8db5 --- /dev/null +++ b/tests/test_template_positions.py @@ -0,0 +1,68 @@ +"""Acceptance tests for issue #288: the positions of a template's filters, includes and +loads are found in time linear in the template. + +Each position was counted from the start of the file, so collecting the filters of a +template applying n of them took time quadratic in its size, where Django compiles and +renders it in linear time. The start of each line of a template is now found once, when +it is read, and each position by bisection. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +TIMEOUT = 60 +COUNT = 200_000 +# The template's number of filters and the last one's line and column, then the number +# of unread entries and the last one. +DESCRIBE = ( + "import json, sys\n" + "from pathlib import Path\n" + "from coretrace_python.taint.templates import project_templates\n" + "found = project_templates(Path(sys.argv[1]))\n" + "filters = found.filters.get('page.html', ())\n" + "last = [filters[-1].location.start_line, filters[-1].location.start_column] if filters else None\n" + "print(json.dumps([len(filters), last, len(found.unread), found.unread[-1:]]))\n" +) + + +@pytest.mark.parametrize( + ("unit", "expected"), + [ + ("{{ v|upper }}", [COUNT, [1, 13 * (COUNT - 1) + 6], 0, []]), + ( + "{% include v %}\n", + [0, None, COUNT, [f"app/templates/page.html:{COUNT} names a template by an expression"]], + ), + ( + "{% load missing %}{{ v|upper }}\n", + [0, None, COUNT, [f"app/templates/page.html:{COUNT} loads 'missing', not found"]], + ), + ], + ids=["filters-on-one-line", "includes-named-by-an-expression", "loads-not-found"], +) +def test_the_positions_in_a_large_template_are_found_in_linear_time( + tmp_path: Path, unit: str, expected: list[object] +) -> None: + # Counted from the start of the file each time, these positions took minutes. + templates = tmp_path / "app" / "templates" + templates.mkdir(parents=True) + (templates / "page.html").write_text(unit * COUNT, encoding="utf-8") + try: + completed = subprocess.run( + [sys.executable, "-c", DESCRIBE, str(tmp_path)], + capture_output=True, + text=True, + check=False, + timeout=TIMEOUT, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"finding the positions did not end within {TIMEOUT} s") + + assert completed.returncode == 0, completed.stderr + assert json.loads(completed.stdout) == expected From 0857bb70182dda22ff607303d447d110f8ed90ec Mon Sep 17 00:00:00 2001 From: Hugo Date: Sun, 1 Nov 2026 01:08:04 +0100 Subject: [PATCH 4/4] docs(changelog): announce the positions found in linear time --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be25874..5fec4c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ - Templates in a cycle of includes are no longer walked again on every path (#284): 20 levels of doubled includes closed into a cycle took 56 s, and 40 templates including one another more than a minute; both now take less than a second. +- The positions in a template are no longer counted from its start, for each filter, + include, extends and load (#288): a template of 2 MB applying 160,000 filters took + 50 s to analyse, and now takes 3 s. ## 0.24.0 (2026-10-23)