Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 39 additions & 30 deletions src/teich/converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
PI_BUILTIN_TOOLS,
)

_INLINE_THINKING_BLOCK_PATTERN = re.compile(r"<(think|thinking)>(.*?)</\1>", re.DOTALL)
FIRST_MESSAGE_TIMESTAMP_METADATA_KEY = "first_message_timestamp"
PI_SYSTEM_PROMPT_CUSTOM_TYPE = "teich-system-prompt"
TEICH_AVAILABLE_TOOLS_CUSTOM_TYPE = "teich-available-tools"
Expand Down Expand Up @@ -1039,27 +1038,16 @@ def _first_text_block(content_blocks: Any) -> str:

def _unwrap_teich_prompt_file(prompt: str) -> str:
normalized = prompt.strip()
match = re.fullmatch(
r'<file\s+name=["\'][^"\']*\.teich-prompt\.txt["\']>\s*(?P<prompt>.*?)\s*</file>',
normalized,
flags=re.DOTALL,
)
if match:
return match.group("prompt")
# Match only the opening tag with a regex: `\s*(.*?)\s*</file>` backtracks cubically on long whitespace runs.
opening = re.match(r'<file\s+name=["\'][^"\']*\.teich-prompt\.txt["\']>', normalized)
if opening and normalized.endswith("</file>"):
return normalized[opening.end() : -len("</file>")].strip()
return prompt


def _split_inline_thinking(text: str) -> tuple[str, str | None]:
reasoning_parts: list[str] = []

def _remove_match(match: re.Match[str]) -> str:
reasoning = match.group(2).strip()
if reasoning:
reasoning_parts.append(reasoning)
return ""

content = _INLINE_THINKING_BLOCK_PATTERN.sub(_remove_match, text).strip()
reasoning = "\n\n".join(reasoning_parts).strip()
blocks, content = _extract_xml_blocks(text, "think", "thinking")
reasoning = "\n\n".join(block for block in blocks if block)
return content, reasoning or None


Expand Down Expand Up @@ -2211,14 +2199,11 @@ def _claude_tool_result_text(block: dict[str, Any]) -> str:


def _claude_local_command_parts(text: str) -> tuple[str, str] | None:
stripped = text.strip()
name_match = re.search(r"<command-name>\s*(?P<name>.*?)\s*</command-name>", stripped, flags=re.DOTALL)
args_match = re.search(r"<command-args>\s*(?P<args>.*?)\s*</command-args>", stripped, flags=re.DOTALL)
if not name_match or not args_match:
names, _ = _extract_xml_blocks(text, "command-name")
args, _ = _extract_xml_blocks(text, "command-args")
if not names or not args:
return None
name = re.sub(r"\s+", " ", name_match.group("name")).strip()
args = args_match.group("args").strip()
return name, args
return re.sub(r"\s+", " ", names[0]), args[0]


def _is_claude_local_command_artifact(text: str) -> bool:
Expand Down Expand Up @@ -3190,11 +3175,35 @@ def _is_hermes_trace_row(value: Any) -> bool:
)


def _extract_xml_blocks(value: str, tag: str) -> tuple[list[str], str]:
pattern = re.compile(rf"<{tag}>\s*(.*?)\s*</{tag}>", re.DOTALL)
blocks = [match.group(1).strip() for match in pattern.finditer(value)]
stripped = pattern.sub("", value).strip()
return blocks, stripped
def _extract_xml_blocks(value: str, *tags: str) -> tuple[list[str], str]:
"""Remove `<tag>...</tag>` blocks (leftmost first, shortest match) and return their stripped contents.

Scans with str.find instead of a `<tag>(.*?)</tag>` regex, which backtracks quadratically or worse on
unclosed tags.
"""
next_open = {tag: value.find(f"<{tag}>") for tag in tags}
blocks: list[str] = []
remaining: list[str] = []
position = 0
while True:
for tag, index in next_open.items():
if 0 <= index < position:
next_open[tag] = value.find(f"<{tag}>", position)
candidates = [(index, tag) for tag, index in next_open.items() if index != -1]
if not candidates:
break
start, tag = min(candidates)
content_start = start + len(tag) + 2
close = value.find(f"</{tag}>", content_start)
if close == -1:
# no closing tag after this one, so no later opening tag of the same name can close either
next_open[tag] = -1
continue
blocks.append(value[content_start:close].strip())
remaining.append(value[position:start])
position = close + len(tag) + 3
remaining.append(value[position:])
return blocks, "".join(remaining).strip()


def _hermes_conversation_to_events(conversation: Any) -> list[dict[str, Any]]:
Expand Down
49 changes: 49 additions & 0 deletions tests/test_converter.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import json
import time
from pathlib import Path

import pytest

from teich import detect_trace_type, validate_tool_calls
from teich.converter import (
_claude_local_command_parts,
_extract_xml_blocks,
_split_inline_thinking,
_unwrap_teich_prompt_file,
convert_trace_to_training_example,
convert_traces_to_training_data,
normalize_claude_code_trace_events,
Expand Down Expand Up @@ -3126,3 +3131,47 @@ def test_codex_skips_injected_runtime_context(tmp_path: Path):
for m in example.messages
for tag in ("<environment_context>", "<user_instructions>")
)


@pytest.mark.parametrize(
("text", "expected"),
[
("a<think>x</think>b<thinking>y</thinking>c", ("abc", "x\n\ny")),
("<think>a<think>b</think>c</think>d", ("c</think>d", "a<think>b")),
("<thinking>a<think>b</think>c</thinking>d", ("d", "a<think>b</think>c")),
("<think>open only", ("<think>open only", None)),
("<think> </think>hi", ("hi", None)),
],
)
def test_split_inline_thinking_takes_leftmost_shortest_blocks(text: str, expected: tuple[str, str | None]):
assert _split_inline_thinking(text) == expected


def test_tag_helpers_keep_regex_semantics():
assert _extract_xml_blocks("pre <tool_response> r </tool_response> post", "tool_response") == (["r"], "pre post")
assert _extract_xml_blocks("<tool_call>{}</tool_call><tool_call>bad", "tool_call") == (["{}"], "<tool_call>bad")
assert _claude_local_command_parts(
"<command-args>a</command-args><command-name> /go\n al </command-name>"
) == ("/go al", "a")
assert _claude_local_command_parts("<command-name>/goal</command-name>") is None
assert _unwrap_teich_prompt_file('<file name="a.teich-prompt.txt">\n hi \n</file>') == "hi"
assert _unwrap_teich_prompt_file('<file name="a.teich-prompt.txt">hi</file> tail') == (
'<file name="a.teich-prompt.txt">hi</file> tail'
)


@pytest.mark.parametrize(
"text",
[
"<think>" * 50_000,
"<think>" + " " * 50_000 + "x",
"<command-name> " * 50_000 + "<command-args>",
'<file name="a.teich-prompt.txt">' + " " * 50_000 + "x",
],
)
def test_tag_parsing_is_linear_on_unclosed_tags(text: str):
start = time.perf_counter()
_split_inline_thinking(text)
_claude_local_command_parts(text)
_unwrap_teich_prompt_file(text)
assert time.perf_counter() - start < 1
Loading