fix(templates): place the blocks of a chain in time linear in its blocks

Each block placed looked up the blocks it holds among all the blocks of its template,
so placing a template's blocks took time quadratic in their number: a 501 KB template
of 16,000 empty blocks took more than 8 s to collect, where main takes 0.07 s, and the
full analysis 11.6 s instead of 0.5 s.

The blocks each block holds, and those standing in no block, are now indexed once per
chain, while the blocks are listed. The same 16,000 blocks are placed in 0.08 s, and
the full analysis takes 0.3 s. What is placed does not change.

Author: SizzleUnrlsd

src/coretrace_python/taint/templates.py

@@ -1099,11 +1099,12 @@ def _page(name: str, readable: Callable[[str | None], _Template | None]) -> _Ren
             break
         current = parent.name
     defining: dict[str, list[int]] = {}
+    held: dict[tuple[int, str | None], list[str]] = {}
     for level, (_, template) in enumerate(chain):
         for block in template.blocks:
             defining.setdefault(block, []).append(level)
-    root = chain[-1][1]
-    pending = [block for block in root.blocks if root.enclosing.get(block) is None] if known else list(defining)
+            held.setdefault((level, template.enclosing.get(block)), []).append(block)
+    pending = list(held.get((len(chain) - 1, None), ())) if known else list(defining)
     placed: set[str] = set()
     shown: set[tuple[int, str]] = set()
     while pending:
@@ -1113,9 +1114,8 @@ def _page(name: str, readable: Callable[[str | None], _Template | None]) -> _Ren
         placed.add(block)
         for level in defining.get(block, []):
             shown.add((level, block))
-            template = chain[level][1]
-            pending.extend(inner for inner in template.blocks if template.enclosing.get(inner) == block)
-            if block not in template.extended:
+            pending.extend(held.get((level, block), ()))
+            if block not in chain[level][1].extended:
                 break
 
     def renders(level: int, blocks: tuple[str, ...]) -> bool:

tests/test_template_block_placement.py

@@ -12,6 +12,9 @@
 
 from __future__ import annotations
 
+import json
+import subprocess
+import sys
 from pathlib import Path
 
 import pytest
@@ -22,6 +25,7 @@
 from coretrace_python.semantic.symbols import SymbolId
 from coretrace_python.taint.templates import project_templates
 
+TIMEOUT = 60
 FILTERS = "python.django.template.defaultfilters"
 STRIPTAGS = f"{FILTERS}.striptags"
 ADVISORY = Advisory(
@@ -207,3 +211,27 @@ def test_a_chain_lists_its_filters_in_the_order_of_a_walk_through_its_extends(tm
     )
 
     assert described(root, "page.html") == [("striptags", "b.html", 1, 6, ("v",)), ("upper", "a.html", 1, 6, ("v",))]
+
+
+COUNT_FILTERS = (
+    "import json, sys\n"
+    "from pathlib import Path\n"
+    "from coretrace_python.taint.templates import project_templates\n"
+    "print(json.dumps({name: len(found) for name, found in project_templates(Path(sys.argv[1])).filters.items()}))\n"
+)
+
+
+def test_the_blocks_of_a_template_are_placed_in_linear_time(tmp_path: Path) -> None:
+    # Looking up the blocks a placed block holds among all the template's blocks made
+    # placing 50,000 blocks take minutes. Only the last one applies a filter.
+    blocks = "".join(f"{{% block b{i} %}}{{% endblock %}}" for i in range(50_000))
+    root = write(tmp_path, {"page.html": blocks + "{% block last %}{{ v|upper }}{% endblock %}"})
+    try:
+        completed = subprocess.run(
+            [sys.executable, "-c", COUNT_FILTERS, str(root)], capture_output=True, text=True, check=False, timeout=TIMEOUT
+        )
+    except subprocess.TimeoutExpired:
+        pytest.fail(f"placing the blocks did not end within {TIMEOUT} s")
+
+    assert completed.returncode == 0, completed.stderr
+    assert json.loads(completed.stdout) == {"page.html": 1}