diff --git a/src/teich/converter.py b/src/teich/converter.py index 2b39ef6..30db112 100644 --- a/src/teich/converter.py +++ b/src/teich/converter.py @@ -21,7 +21,6 @@ PI_BUILTIN_TOOLS, ) -_INLINE_THINKING_BLOCK_PATTERN = re.compile(r"<(think|thinking)>(.*?)", re.DOTALL) FIRST_MESSAGE_TIMESTAMP_METADATA_KEY = "first_message_timestamp" PI_SYSTEM_PROMPT_CUSTOM_TYPE = "teich-system-prompt" TEICH_AVAILABLE_TOOLS_CUSTOM_TYPE = "teich-available-tools" @@ -1039,27 +1038,16 @@ def _first_text_block(content_blocks: Any) -> str: def _unwrap_teich_prompt_file(prompt: str) -> str: normalized = prompt.strip() - match = re.fullmatch( - r'\s*(?P.*?)\s*', - normalized, - flags=re.DOTALL, - ) - if match: - return match.group("prompt") + # Match only the opening tag with a regex: `\s*(.*?)\s*` backtracks cubically on long whitespace runs. + opening = re.match(r'', normalized) + if opening and normalized.endswith(""): + return normalized[opening.end() : -len("")].strip() return prompt def _split_inline_thinking(text: str) -> tuple[str, str | None]: - reasoning_parts: list[str] = [] - - def _remove_match(match: re.Match[str]) -> str: - reasoning = match.group(2).strip() - if reasoning: - reasoning_parts.append(reasoning) - return "" - - content = _INLINE_THINKING_BLOCK_PATTERN.sub(_remove_match, text).strip() - reasoning = "\n\n".join(reasoning_parts).strip() + blocks, content = _extract_xml_blocks(text, "think", "thinking") + reasoning = "\n\n".join(block for block in blocks if block) return content, reasoning or None @@ -2211,14 +2199,11 @@ def _claude_tool_result_text(block: dict[str, Any]) -> str: def _claude_local_command_parts(text: str) -> tuple[str, str] | None: - stripped = text.strip() - name_match = re.search(r"\s*(?P.*?)\s*", stripped, flags=re.DOTALL) - args_match = re.search(r"\s*(?P.*?)\s*", stripped, flags=re.DOTALL) - if not name_match or not args_match: + names, _ = _extract_xml_blocks(text, "command-name") + args, _ = _extract_xml_blocks(text, "command-args") + if not names or not args: return None - name = re.sub(r"\s+", " ", name_match.group("name")).strip() - args = args_match.group("args").strip() - return name, args + return re.sub(r"\s+", " ", names[0]), args[0] def _is_claude_local_command_artifact(text: str) -> bool: @@ -3190,11 +3175,35 @@ def _is_hermes_trace_row(value: Any) -> bool: ) -def _extract_xml_blocks(value: str, tag: str) -> tuple[list[str], str]: - pattern = re.compile(rf"<{tag}>\s*(.*?)\s*", re.DOTALL) - blocks = [match.group(1).strip() for match in pattern.finditer(value)] - stripped = pattern.sub("", value).strip() - return blocks, stripped +def _extract_xml_blocks(value: str, *tags: str) -> tuple[list[str], str]: + """Remove `...` blocks (leftmost first, shortest match) and return their stripped contents. + + Scans with str.find instead of a `(.*?)` regex, which backtracks quadratically or worse on + unclosed tags. + """ + next_open = {tag: value.find(f"<{tag}>") for tag in tags} + blocks: list[str] = [] + remaining: list[str] = [] + position = 0 + while True: + for tag, index in next_open.items(): + if 0 <= index < position: + next_open[tag] = value.find(f"<{tag}>", position) + candidates = [(index, tag) for tag, index in next_open.items() if index != -1] + if not candidates: + break + start, tag = min(candidates) + content_start = start + len(tag) + 2 + close = value.find(f"", content_start) + if close == -1: + # no closing tag after this one, so no later opening tag of the same name can close either + next_open[tag] = -1 + continue + blocks.append(value[content_start:close].strip()) + remaining.append(value[position:start]) + position = close + len(tag) + 3 + remaining.append(value[position:]) + return blocks, "".join(remaining).strip() def _hermes_conversation_to_events(conversation: Any) -> list[dict[str, Any]]: diff --git a/tests/test_converter.py b/tests/test_converter.py index 2f0fe0d..9fd0a07 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -1,10 +1,15 @@ import json +import time from pathlib import Path import pytest from teich import detect_trace_type, validate_tool_calls from teich.converter import ( + _claude_local_command_parts, + _extract_xml_blocks, + _split_inline_thinking, + _unwrap_teich_prompt_file, convert_trace_to_training_example, convert_traces_to_training_data, normalize_claude_code_trace_events, @@ -3126,3 +3131,47 @@ def test_codex_skips_injected_runtime_context(tmp_path: Path): for m in example.messages for tag in ("", "") ) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("axbyc", ("abc", "x\n\ny")), + ("abcd", ("cd", "ab")), + ("abcd", ("d", "abc")), + ("open only", ("open only", None)), + (" hi", ("hi", None)), + ], +) +def test_split_inline_thinking_takes_leftmost_shortest_blocks(text: str, expected: tuple[str, str | None]): + assert _split_inline_thinking(text) == expected + + +def test_tag_helpers_keep_regex_semantics(): + assert _extract_xml_blocks("pre r post", "tool_response") == (["r"], "pre post") + assert _extract_xml_blocks("{}bad", "tool_call") == (["{}"], "bad") + assert _claude_local_command_parts( + "a /go\n al " + ) == ("/go al", "a") + assert _claude_local_command_parts("/goal") is None + assert _unwrap_teich_prompt_file('\n hi \n') == "hi" + assert _unwrap_teich_prompt_file('hi tail') == ( + 'hi tail' + ) + + +@pytest.mark.parametrize( + "text", + [ + "" * 50_000, + "" + " " * 50_000 + "x", + " " * 50_000 + "", + '' + " " * 50_000 + "x", + ], +) +def test_tag_parsing_is_linear_on_unclosed_tags(text: str): + start = time.perf_counter() + _split_inline_thinking(text) + _claude_local_command_parts(text) + _unwrap_teich_prompt_file(text) + assert time.perf_counter() - start < 1