diff --git a/src/teich/converter.py b/src/teich/converter.py
index 2b39ef6..30db112 100644
--- a/src/teich/converter.py
+++ b/src/teich/converter.py
@@ -21,7 +21,6 @@
PI_BUILTIN_TOOLS,
)
-_INLINE_THINKING_BLOCK_PATTERN = re.compile(r"<(think|thinking)>(.*?)\1>", re.DOTALL)
FIRST_MESSAGE_TIMESTAMP_METADATA_KEY = "first_message_timestamp"
PI_SYSTEM_PROMPT_CUSTOM_TYPE = "teich-system-prompt"
TEICH_AVAILABLE_TOOLS_CUSTOM_TYPE = "teich-available-tools"
@@ -1039,27 +1038,16 @@ def _first_text_block(content_blocks: Any) -> str:
def _unwrap_teich_prompt_file(prompt: str) -> str:
normalized = prompt.strip()
- match = re.fullmatch(
- r'\s*(?P.*?)\s*',
- normalized,
- flags=re.DOTALL,
- )
- if match:
- return match.group("prompt")
+ # Match only the opening tag with a regex: `\s*(.*?)\s*` backtracks cubically on long whitespace runs.
+ opening = re.match(r'', normalized)
+ if opening and normalized.endswith(""):
+ return normalized[opening.end() : -len("")].strip()
return prompt
def _split_inline_thinking(text: str) -> tuple[str, str | None]:
- reasoning_parts: list[str] = []
-
- def _remove_match(match: re.Match[str]) -> str:
- reasoning = match.group(2).strip()
- if reasoning:
- reasoning_parts.append(reasoning)
- return ""
-
- content = _INLINE_THINKING_BLOCK_PATTERN.sub(_remove_match, text).strip()
- reasoning = "\n\n".join(reasoning_parts).strip()
+ blocks, content = _extract_xml_blocks(text, "think", "thinking")
+ reasoning = "\n\n".join(block for block in blocks if block)
return content, reasoning or None
@@ -2211,14 +2199,11 @@ def _claude_tool_result_text(block: dict[str, Any]) -> str:
def _claude_local_command_parts(text: str) -> tuple[str, str] | None:
- stripped = text.strip()
- name_match = re.search(r"\s*(?P.*?)\s*", stripped, flags=re.DOTALL)
- args_match = re.search(r"\s*(?P.*?)\s*", stripped, flags=re.DOTALL)
- if not name_match or not args_match:
+ names, _ = _extract_xml_blocks(text, "command-name")
+ args, _ = _extract_xml_blocks(text, "command-args")
+ if not names or not args:
return None
- name = re.sub(r"\s+", " ", name_match.group("name")).strip()
- args = args_match.group("args").strip()
- return name, args
+ return re.sub(r"\s+", " ", names[0]), args[0]
def _is_claude_local_command_artifact(text: str) -> bool:
@@ -3190,11 +3175,35 @@ def _is_hermes_trace_row(value: Any) -> bool:
)
-def _extract_xml_blocks(value: str, tag: str) -> tuple[list[str], str]:
- pattern = re.compile(rf"<{tag}>\s*(.*?)\s*{tag}>", re.DOTALL)
- blocks = [match.group(1).strip() for match in pattern.finditer(value)]
- stripped = pattern.sub("", value).strip()
- return blocks, stripped
+def _extract_xml_blocks(value: str, *tags: str) -> tuple[list[str], str]:
+ """Remove `...` blocks (leftmost first, shortest match) and return their stripped contents.
+
+ Scans with str.find instead of a `(.*?)` regex, which backtracks quadratically or worse on
+ unclosed tags.
+ """
+ next_open = {tag: value.find(f"<{tag}>") for tag in tags}
+ blocks: list[str] = []
+ remaining: list[str] = []
+ position = 0
+ while True:
+ for tag, index in next_open.items():
+ if 0 <= index < position:
+ next_open[tag] = value.find(f"<{tag}>", position)
+ candidates = [(index, tag) for tag, index in next_open.items() if index != -1]
+ if not candidates:
+ break
+ start, tag = min(candidates)
+ content_start = start + len(tag) + 2
+ close = value.find(f"{tag}>", content_start)
+ if close == -1:
+ # no closing tag after this one, so no later opening tag of the same name can close either
+ next_open[tag] = -1
+ continue
+ blocks.append(value[content_start:close].strip())
+ remaining.append(value[position:start])
+ position = close + len(tag) + 3
+ remaining.append(value[position:])
+ return blocks, "".join(remaining).strip()
def _hermes_conversation_to_events(conversation: Any) -> list[dict[str, Any]]:
diff --git a/tests/test_converter.py b/tests/test_converter.py
index 2f0fe0d..9fd0a07 100644
--- a/tests/test_converter.py
+++ b/tests/test_converter.py
@@ -1,10 +1,15 @@
import json
+import time
from pathlib import Path
import pytest
from teich import detect_trace_type, validate_tool_calls
from teich.converter import (
+ _claude_local_command_parts,
+ _extract_xml_blocks,
+ _split_inline_thinking,
+ _unwrap_teich_prompt_file,
convert_trace_to_training_example,
convert_traces_to_training_data,
normalize_claude_code_trace_events,
@@ -3126,3 +3131,47 @@ def test_codex_skips_injected_runtime_context(tmp_path: Path):
for m in example.messages
for tag in ("", "")
)
+
+
+@pytest.mark.parametrize(
+ ("text", "expected"),
+ [
+ ("axbyc", ("abc", "x\n\ny")),
+ ("abcd", ("cd", "ab")),
+ ("abcd", ("d", "abc")),
+ ("open only", ("open only", None)),
+ (" hi", ("hi", None)),
+ ],
+)
+def test_split_inline_thinking_takes_leftmost_shortest_blocks(text: str, expected: tuple[str, str | None]):
+ assert _split_inline_thinking(text) == expected
+
+
+def test_tag_helpers_keep_regex_semantics():
+ assert _extract_xml_blocks("pre r post", "tool_response") == (["r"], "pre post")
+ assert _extract_xml_blocks("{}bad", "tool_call") == (["{}"], "bad")
+ assert _claude_local_command_parts(
+ "a /go\n al "
+ ) == ("/go al", "a")
+ assert _claude_local_command_parts("/goal") is None
+ assert _unwrap_teich_prompt_file('\n hi \n') == "hi"
+ assert _unwrap_teich_prompt_file('hi tail') == (
+ 'hi tail'
+ )
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ "" * 50_000,
+ "" + " " * 50_000 + "x",
+ " " * 50_000 + "",
+ '' + " " * 50_000 + "x",
+ ],
+)
+def test_tag_parsing_is_linear_on_unclosed_tags(text: str):
+ start = time.perf_counter()
+ _split_inline_thinking(text)
+ _claude_local_command_parts(text)
+ _unwrap_teich_prompt_file(text)
+ assert time.perf_counter() - start < 1