diff --git a/.changeset/fix-pi-coding-agent-stream-function.md b/.changeset/fix-pi-coding-agent-stream-function.md new file mode 100644 index 000000000..667ca8321 --- /dev/null +++ b/.changeset/fix-pi-coding-agent-stream-function.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(pi-coding-agent): Instrument `streamFunction` properly diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index ba0c67362..3b2ac8370 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -94,7 +94,15 @@ }, { "variantKey": "pi-coding-agent-v0-latest", - "label": "v0 latest" + "label": "latest" + }, + { + "variantKey": "pi-coding-agent-v081", + "label": "v0.81 pinned" + }, + { + "variantKey": "pi-coding-agent-v081-latest", + "label": "v0.81 latest" } ] }, diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0-latest.cassette.json b/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0-latest.cassette.json deleted file mode 100644 index f28ebe7c7..000000000 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__cassettes__/pi-coding-agent-v0-latest.cassette.json +++ /dev/null @@ -1,264 +0,0 @@ -{ - "entries": [ - { - "callIndex": 0, - "id": "5625e915bcb72d05", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-07-06T18:58:14.490Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "type": "text" - } - ], - "role": "user" - } - ], - "model": "claude-haiku-4-5", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: 2026-07-06\nCurrent working directory: /workspace/e2e/.bt-tmp/run-65wF9N/scenarios/pi-coding-agent-instrumentation", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds (optional, no default timeout)", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01FtZyPsifurKbjrr7QMZxsj\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":2261,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01DebxZNR5SkiKGnFt4YDJqg\",\"name\":\"bash\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"command\\\": \\\"printf pi_tool_ok\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":2261,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":57}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "10000000", - "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T18:58:13Z", - "anthropic-ratelimit-output-tokens-limit": "2000000", - "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T18:58:13Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T18:58:13Z", - "anthropic-ratelimit-tokens-limit": "12000000", - "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T18:58:13Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a170d6f3be5fcb7b-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 06 Jul 2026 18:58:14 GMT", - "request-id": "req_011CcmKJh7wudjLNVptxcJNa", - "server": "cloudflare", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-82e56324cdf78fa2b0c7b53f35964f0e-9c7b152968529e42-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 1, - "id": "0c74acf5b76aad36", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-07-06T18:58:15.517Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01DebxZNR5SkiKGnFt4YDJqg", - "input": { - "command": "printf pi_tool_ok" - }, - "name": "bash", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "pi_tool_ok", - "is_error": false, - "tool_use_id": "toolu_01DebxZNR5SkiKGnFt4YDJqg", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-haiku-4-5", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /workspace/e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-a0beda3ff342f9cd/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\n\n\n\nProject-specific instructions and guidelines:\n\n\n# Braintrust JavaScript SDK Monorepo\n\nTypeScript SDKs and integrations for Braintrust. Uses `pnpm` workspaces.\n\n## Repository Structure\n\n```text\n.\n├── js/ # Main `braintrust` package\n├── integrations/ # Integration packages (@braintrust/*)\n├── e2e/ # End-to-end scenario tests (mock server + subprocess isolation)\n├── docs/ # Docs and reference material\n└── internal/ # Internal test fixtures and golden projects\n```\n\n## Setup\n\n```bash\nmise install # Install toolchain and dependencies\n```\n\n## Build\n\n```bash\npnpm run build # Build all workspace packages (from repo root)\n```\n\n## Instrumentation\n\nUse the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch.\n\n## Testing\n\nUses Vitest. Prefer running the **narrowest relevant test** rather than the full suite.\n\n**From `js/` directory:**\n\n```bash\npnpm test # Core vitest suite (excludes wrappers)\npnpm test -- -t \"test name\" # Filter by test name\npnpm run test:checks # Hermetic tests (core + vitest wrapper)\n```\n\n**E2E tests (`e2e/`):**\n\nEach scenario runs the SDK in a subprocess against a mock Braintrust server and snapshots the results. No API keys required for replay; recording needs provider keys.\n\n```bash\npnpm run test:e2e # Run all e2e scenarios (from repo root)\npnpm run test:e2e:update # Update e2e snapshots without re-recording cassettes\npnpm run test:e2e:record # Re-record provider cassettes and update snapshots\n```\n\nWhen adding or modifying e2e tests, run the relevant e2e verification twice before stopping so flakes are caught proactively. After running `pnpm run test:e2e:update` or `pnpm run test:e2e:record`, always run the normal e2e tests afterward to verify there is no snapshot drift or unstable output.\n\nSpan-tree snapshots are paired: `*.span-tree.json` is the structural contract, and `*.span-tree.txt` is the human-readable ASCII tree generated from the same normalized spans. Both files are asserted and should be updated together through `pnpm run test:e2e:update` or `pnpm run test:e2e:record`; do not hand-edit only one side of the pair.\n\n**From repo root:**\n\n```bash\npnpm run test # Run all workspace tests via turbo\n```\n\n## Linting & Formatting\n\nRun from the repo root. **Always run `fix:formatting` before committing** — there is a pre-commit hook that will reject unformatted code.\n\n```bash\npnpm run formatting # Check formatting (prettier)\npnpm run lint # Run eslint checks\npnpm run fix:formatting # Auto-fix formatting\npnpm run fix:lint # Auto-fix eslint issues\n```\n\n## Vendored Forks and Licenses\n\nWhen touching forked third-party code in this repository, including the\nvendored `import-in-the-middle`, `require-in-the-middle`, or `orchestrion-js`\ncode, preserve and respect the upstream license requirements. Keep copyright\nnotices, license files, provenance notes, and `js/NOTICE` entries accurate when\ncopying, updating, or materially modifying vendored code.\n\n\n\n\n\nCurrent date: 2026-07-06\nCurrent working directory: /workspace/e2e/.bt-tmp/run-65wF9N/scenarios/pi-coding-agent-instrumentation", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds (optional, no default timeout)", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01THP1NeVrahpg1VDncn2oNL\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":2335,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"PI\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"_CODING_AGENT_OK\\n\\nCommand output: `pi_tool_ok`\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":2335,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":23}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "10000000", - "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T18:58:14Z", - "anthropic-ratelimit-output-tokens-limit": "2000000", - "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T18:58:14Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T18:58:14Z", - "anthropic-ratelimit-tokens-limit": "12000000", - "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T18:58:14Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a170d6f97e51cb7b-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 06 Jul 2026 18:58:15 GMT", - "request-id": "req_011CcmKJm3q6ifiuWTEzX3Nx", - "server": "cloudflare", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-66f454e0bc80b7349730ee9d00eca736-34204b0eac2afcb1-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - } - ], - "meta": { - "createdAt": "2026-07-06T14:00:38.142Z" - } -} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json index 7569fe54b..c31083484 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { @@ -51,7 +51,7 @@ } ], "role": "tool", - "tool_call_id": "" + "tool_call_id": "tool-call-1" } ], "output": [ @@ -59,7 +59,7 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", + "content": "PI_CODING_AGENT_OK pi_tool_ok", "role": "assistant" } } @@ -68,18 +68,18 @@ "model": "claude-haiku-4-5", "pi_coding_agent.api": "anthropic-messages", "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "agent.streamFn", + "pi_coding_agent.operation": "agent.streamFunction", "pi_coding_agent.stop_reason": "stop", "provider": "anthropic" }, "metrics": { - "completion_tokens": 23, + "completion_tokens": 3, "duration": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 2335, + "prompt_tokens": 5, "time_to_first_token": 0, - "tokens": 2358 + "tokens": 8 } }, { @@ -98,7 +98,7 @@ ] }, "metadata": { - "gen_ai.tool.call.id": "", + "gen_ai.tool.call.id": "tool-call-1", "gen_ai.tool.name": "bash", "pi_coding_agent.tool.name": "bash" } @@ -110,7 +110,7 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", + "content": "PI_CODING_AGENT_OK pi_tool_ok", "role": "assistant" } } @@ -124,12 +124,12 @@ "provider": "anthropic" }, "metrics": { - "completion_tokens": 80, + "completion_tokens": 6, "duration": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 4596, - "tokens": 4676 + "prompt_tokens": 10, + "tokens": 16 } } ], diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt index 4db394c56..8e0c777bd 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-auto-hook.span-tree.txt @@ -14,7 +14,7 @@ span_tree: "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", + "content": "PI_CODING_AGENT_OK pi_tool_ok", "role": "assistant" } } @@ -28,17 +28,17 @@ span_tree: "provider": "anthropic" } metrics: { - "completion_tokens": 80, + "completion_tokens": 6, "duration": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 4596, - "tokens": 4676 + "prompt_tokens": 10, + "tokens": 16 } ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { @@ -72,7 +72,7 @@ span_tree: │ } │ ], │ "role": "tool", - │ "tool_call_id": "" + │ "tool_call_id": "tool-call-1" │ } │ ] │ output: [ @@ -80,7 +80,7 @@ span_tree: │ "finish_reason": "stop", │ "index": 0, │ "message": { - │ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", + │ "content": "PI_CODING_AGENT_OK pi_tool_ok", │ "role": "assistant" │ } │ } @@ -89,18 +89,18 @@ span_tree: │ "model": "claude-haiku-4-5", │ "pi_coding_agent.api": "anthropic-messages", │ "pi_coding_agent.model": "claude-haiku-4-5", - │ "pi_coding_agent.operation": "agent.streamFn", + │ "pi_coding_agent.operation": "agent.streamFunction", │ "pi_coding_agent.stop_reason": "stop", │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 23, + │ "completion_tokens": 3, │ "duration": 0, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 2335, + │ "prompt_tokens": 5, │ "time_to_first_token": 0, - │ "tokens": 2358 + │ "tokens": 8 │ } └── bash [tool] input: { @@ -115,7 +115,7 @@ span_tree: ] } metadata: { - "gen_ai.tool.call.id": "", + "gen_ai.tool.call.id": "tool-call-1", "gen_ai.tool.name": "bash", "pi_coding_agent.tool.name": "bash" } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json index 7569fe54b..c31083484 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.json @@ -17,7 +17,7 @@ "children": [], "input": [ { - "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", "role": "system" }, { @@ -51,7 +51,7 @@ } ], "role": "tool", - "tool_call_id": "" + "tool_call_id": "tool-call-1" } ], "output": [ @@ -59,7 +59,7 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", + "content": "PI_CODING_AGENT_OK pi_tool_ok", "role": "assistant" } } @@ -68,18 +68,18 @@ "model": "claude-haiku-4-5", "pi_coding_agent.api": "anthropic-messages", "pi_coding_agent.model": "claude-haiku-4-5", - "pi_coding_agent.operation": "agent.streamFn", + "pi_coding_agent.operation": "agent.streamFunction", "pi_coding_agent.stop_reason": "stop", "provider": "anthropic" }, "metrics": { - "completion_tokens": 23, + "completion_tokens": 3, "duration": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 2335, + "prompt_tokens": 5, "time_to_first_token": 0, - "tokens": 2358 + "tokens": 8 } }, { @@ -98,7 +98,7 @@ ] }, "metadata": { - "gen_ai.tool.call.id": "", + "gen_ai.tool.call.id": "tool-call-1", "gen_ai.tool.name": "bash", "pi_coding_agent.tool.name": "bash" } @@ -110,7 +110,7 @@ "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", + "content": "PI_CODING_AGENT_OK pi_tool_ok", "role": "assistant" } } @@ -124,12 +124,12 @@ "provider": "anthropic" }, "metrics": { - "completion_tokens": 80, + "completion_tokens": 6, "duration": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 4596, - "tokens": 4676 + "prompt_tokens": 10, + "tokens": 16 } } ], diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt index 4db394c56..8e0c777bd 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v0-latest-wrapped.span-tree.txt @@ -14,7 +14,7 @@ span_tree: "finish_reason": "stop", "index": 0, "message": { - "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", + "content": "PI_CODING_AGENT_OK pi_tool_ok", "role": "assistant" } } @@ -28,17 +28,17 @@ span_tree: "provider": "anthropic" } metrics: { - "completion_tokens": 80, + "completion_tokens": 6, "duration": 0, "prompt_cache_creation_tokens": 0, "prompt_cached_tokens": 0, - "prompt_tokens": 4596, - "tokens": 4676 + "prompt_tokens": 10, + "tokens": 16 } ├── anthropic.messages.create [llm] │ input: [ │ { - │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.79.10_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent date: \nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- You can inspect PI_* environment variables for current model and session details.\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.84.3_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", │ "role": "system" │ }, │ { @@ -72,7 +72,7 @@ span_tree: │ } │ ], │ "role": "tool", - │ "tool_call_id": "" + │ "tool_call_id": "tool-call-1" │ } │ ] │ output: [ @@ -80,7 +80,7 @@ span_tree: │ "finish_reason": "stop", │ "index": 0, │ "message": { - │ "content": "PI_CODING_AGENT_OK\n\nCommand output: `pi_tool_ok`", + │ "content": "PI_CODING_AGENT_OK pi_tool_ok", │ "role": "assistant" │ } │ } @@ -89,18 +89,18 @@ span_tree: │ "model": "claude-haiku-4-5", │ "pi_coding_agent.api": "anthropic-messages", │ "pi_coding_agent.model": "claude-haiku-4-5", - │ "pi_coding_agent.operation": "agent.streamFn", + │ "pi_coding_agent.operation": "agent.streamFunction", │ "pi_coding_agent.stop_reason": "stop", │ "provider": "anthropic" │ } │ metrics: { - │ "completion_tokens": 23, + │ "completion_tokens": 3, │ "duration": 0, │ "prompt_cache_creation_tokens": 0, │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 2335, + │ "prompt_tokens": 5, │ "time_to_first_token": 0, - │ "tokens": 2358 + │ "tokens": 8 │ } └── bash [tool] input: { @@ -115,7 +115,7 @@ span_tree: ] } metadata: { - "gen_ai.tool.call.id": "", + "gen_ai.tool.call.id": "tool-call-1", "gen_ai.tool.name": "bash", "pi_coding_agent.tool.name": "bash" } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.json new file mode 100644 index 000000000..53abefa21 --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.json @@ -0,0 +1,146 @@ +{ + "span_tree": [ + { + "name": "pi-coding-agent-root", + "type": "task", + "children": [ + { + "name": "pi-coding-agent-prompt-operation", + "children": [ + { + "name": "AgentSession.prompt", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + }, + { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ], + "role": "tool", + "tool_call_id": "tool-call-1" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.stop_reason": "stop", + "provider": "anthropic" + }, + "metrics": { + "completion_tokens": 3, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 8 + } + }, + { + "name": "bash", + "type": "tool", + "children": [], + "input": { + "command": "printf pi_tool_ok" + }, + "output": { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ] + }, + "metadata": { + "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.name": "bash", + "pi_coding_agent.tool.name": "bash" + } + } + ], + "input": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.source": "rpc", + "provider": "anthropic" + }, + "metrics": { + "completion_tokens": 6, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "prompt" + } + } + ], + "metadata": { + "scenario": "pi-coding-agent-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.txt new file mode 100644 index 000000000..7e558489c --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-auto-hook.span-tree.txt @@ -0,0 +1,121 @@ +span_tree: +└── pi-coding-agent-root [task] + metadata: { + "scenario": "pi-coding-agent-instrumentation" + } + └── pi-coding-agent-prompt-operation + metadata: { + "operation": "prompt" + } + └── AgentSession.prompt [task] + input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." + output: [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ] + metadata: { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.source": "rpc", + "provider": "anthropic" + } + metrics: { + "completion_tokens": 6, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "tokens": 16 + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + │ "name": "bash" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ }, + │ { + │ "content": [ + │ { + │ "text": "pi_tool_ok", + │ "type": "text" + │ } + │ ], + │ "role": "tool", + │ "tool_call_id": "tool-call-1" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "PI_CODING_AGENT_OK pi_tool_ok", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "pi_coding_agent.api": "anthropic-messages", + │ "pi_coding_agent.model": "claude-haiku-4-5", + │ "pi_coding_agent.operation": "agent.streamFunction", + │ "pi_coding_agent.stop_reason": "stop", + │ "provider": "anthropic" + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "duration": 0, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 8 + │ } + └── bash [tool] + input: { + "command": "printf pi_tool_ok" + } + output: { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ] + } + metadata: { + "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.name": "bash", + "pi_coding_agent.tool.name": "bash" + } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.json new file mode 100644 index 000000000..f17ea9fe7 --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.json @@ -0,0 +1,146 @@ +{ + "span_tree": [ + { + "name": "pi-coding-agent-root", + "type": "task", + "children": [ + { + "name": "pi-coding-agent-prompt-operation", + "children": [ + { + "name": "AgentSession.prompt", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + }, + { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ], + "role": "tool", + "tool_call_id": "tool-call-1" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.stop_reason": "stop", + "provider": "anthropic" + }, + "metrics": { + "completion_tokens": 3, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 8 + } + }, + { + "name": "bash", + "type": "tool", + "children": [], + "input": { + "command": "printf pi_tool_ok" + }, + "output": { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ] + }, + "metadata": { + "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.name": "bash", + "pi_coding_agent.tool.name": "bash" + } + } + ], + "input": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.source": "rpc", + "provider": "anthropic" + }, + "metrics": { + "completion_tokens": 6, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "prompt" + } + } + ], + "metadata": { + "scenario": "pi-coding-agent-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.txt new file mode 100644 index 000000000..b7643d1ce --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-auto-hook.span-tree.txt @@ -0,0 +1,121 @@ +span_tree: +└── pi-coding-agent-root [task] + metadata: { + "scenario": "pi-coding-agent-instrumentation" + } + └── pi-coding-agent-prompt-operation + metadata: { + "operation": "prompt" + } + └── AgentSession.prompt [task] + input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." + output: [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ] + metadata: { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.source": "rpc", + "provider": "anthropic" + } + metrics: { + "completion_tokens": 6, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "tokens": 16 + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + │ "name": "bash" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ }, + │ { + │ "content": [ + │ { + │ "text": "pi_tool_ok", + │ "type": "text" + │ } + │ ], + │ "role": "tool", + │ "tool_call_id": "tool-call-1" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "PI_CODING_AGENT_OK pi_tool_ok", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "pi_coding_agent.api": "anthropic-messages", + │ "pi_coding_agent.model": "claude-haiku-4-5", + │ "pi_coding_agent.operation": "agent.streamFunction", + │ "pi_coding_agent.stop_reason": "stop", + │ "provider": "anthropic" + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "duration": 0, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 8 + │ } + └── bash [tool] + input: { + "command": "printf pi_tool_ok" + } + output: { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ] + } + metadata: { + "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.name": "bash", + "pi_coding_agent.tool.name": "bash" + } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.json new file mode 100644 index 000000000..f17ea9fe7 --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.json @@ -0,0 +1,146 @@ +{ + "span_tree": [ + { + "name": "pi-coding-agent-root", + "type": "task", + "children": [ + { + "name": "pi-coding-agent-prompt-operation", + "children": [ + { + "name": "AgentSession.prompt", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + }, + { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ], + "role": "tool", + "tool_call_id": "tool-call-1" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.stop_reason": "stop", + "provider": "anthropic" + }, + "metrics": { + "completion_tokens": 3, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 8 + } + }, + { + "name": "bash", + "type": "tool", + "children": [], + "input": { + "command": "printf pi_tool_ok" + }, + "output": { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ] + }, + "metadata": { + "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.name": "bash", + "pi_coding_agent.tool.name": "bash" + } + } + ], + "input": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.source": "rpc", + "provider": "anthropic" + }, + "metrics": { + "completion_tokens": 6, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "prompt" + } + } + ], + "metadata": { + "scenario": "pi-coding-agent-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.txt new file mode 100644 index 000000000..b7643d1ce --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-latest-wrapped.span-tree.txt @@ -0,0 +1,121 @@ +span_tree: +└── pi-coding-agent-root [task] + metadata: { + "scenario": "pi-coding-agent-instrumentation" + } + └── pi-coding-agent-prompt-operation + metadata: { + "operation": "prompt" + } + └── AgentSession.prompt [task] + input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." + output: [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ] + metadata: { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.source": "rpc", + "provider": "anthropic" + } + metrics: { + "completion_tokens": 6, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "tokens": 16 + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.1_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + │ "name": "bash" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ }, + │ { + │ "content": [ + │ { + │ "text": "pi_tool_ok", + │ "type": "text" + │ } + │ ], + │ "role": "tool", + │ "tool_call_id": "tool-call-1" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "PI_CODING_AGENT_OK pi_tool_ok", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "pi_coding_agent.api": "anthropic-messages", + │ "pi_coding_agent.model": "claude-haiku-4-5", + │ "pi_coding_agent.operation": "agent.streamFunction", + │ "pi_coding_agent.stop_reason": "stop", + │ "provider": "anthropic" + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "duration": 0, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 8 + │ } + └── bash [tool] + input: { + "command": "printf pi_tool_ok" + } + output: { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ] + } + metadata: { + "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.name": "bash", + "pi_coding_agent.tool.name": "bash" + } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.json b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.json new file mode 100644 index 000000000..53abefa21 --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.json @@ -0,0 +1,146 @@ +{ + "span_tree": [ + { + "name": "pi-coding-agent-root", + "type": "task", + "children": [ + { + "name": "pi-coding-agent-prompt-operation", + "children": [ + { + "name": "AgentSession.prompt", + "type": "task", + "children": [ + { + "name": "anthropic.messages.create", + "type": "llm", + "children": [], + "input": [ + { + "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + "role": "system" + }, + { + "content": [ + { + "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "type": "text" + } + ], + "role": "user" + }, + { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"command\":\"printf pi_tool_ok\"}", + "name": "bash" + }, + "id": "", + "type": "function" + } + ] + }, + { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ], + "role": "tool", + "tool_call_id": "tool-call-1" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "agent.streamFunction", + "pi_coding_agent.stop_reason": "stop", + "provider": "anthropic" + }, + "metrics": { + "completion_tokens": 3, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 5, + "time_to_first_token": 0, + "tokens": 8 + } + }, + { + "name": "bash", + "type": "tool", + "children": [], + "input": { + "command": "printf pi_tool_ok" + }, + "output": { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ] + }, + "metadata": { + "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.name": "bash", + "pi_coding_agent.tool.name": "bash" + } + } + ], + "input": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + "output": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ], + "metadata": { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.source": "rpc", + "provider": "anthropic" + }, + "metrics": { + "completion_tokens": 6, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "tokens": 16 + } + } + ], + "metadata": { + "operation": "prompt" + } + } + ], + "metadata": { + "scenario": "pi-coding-agent-instrumentation" + } + } + ] +} diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.txt b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.txt new file mode 100644 index 000000000..7e558489c --- /dev/null +++ b/e2e/scenarios/pi-coding-agent-instrumentation/__snapshots__/pi-coding-agent-v081-wrapped.span-tree.txt @@ -0,0 +1,121 @@ +span_tree: +└── pi-coding-agent-root [task] + metadata: { + "scenario": "pi-coding-agent-instrumentation" + } + └── pi-coding-agent-prompt-operation + metadata: { + "operation": "prompt" + } + └── AgentSession.prompt [task] + input: "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output." + output: [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "PI_CODING_AGENT_OK pi_tool_ok", + "role": "assistant" + } + } + ] + metadata: { + "model": "claude-haiku-4-5", + "pi_coding_agent.api": "anthropic-messages", + "pi_coding_agent.model": "claude-haiku-4-5", + "pi_coding_agent.operation": "AgentSession.prompt", + "pi_coding_agent.source": "rpc", + "provider": "anthropic" + } + metrics: { + "completion_tokens": 6, + "duration": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 10, + "tokens": 16 + } + ├── anthropic.messages.create [llm] + │ input: [ + │ { + │ "content": "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files.\n\nAvailable tools:\n- bash: Execute bash commands (ls, grep, find, etc.)\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n\nGuidelines:\n- Use bash for file operations like ls, rg, find\n- Be concise in your responses\n- Show file paths clearly when working with files\n\nPi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI):\n- Main documentation: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/README.md\n- Additional docs: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/docs\n- Examples: /e2e/.bt-tmp/scenario-deps/pi-coding-agent-instrumentation-locked-/node_modules/.pnpm/@earendil-works+pi-coding-agent@0.81.0_ws@8.21.0_zod@4.4.3/node_modules/@earendil-works/pi-coding-agent/examples (extensions, custom tools, SDK)\n- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory\n- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md)\n- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing\n- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)\nCurrent working directory: /e2e/scenarios/pi-coding-agent-instrumentation/", + │ "role": "system" + │ }, + │ { + │ "content": [ + │ { + │ "text": "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", + │ "type": "text" + │ } + │ ], + │ "role": "user" + │ }, + │ { + │ "content": null, + │ "role": "assistant", + │ "tool_calls": [ + │ { + │ "function": { + │ "arguments": "{\"command\":\"printf pi_tool_ok\"}", + │ "name": "bash" + │ }, + │ "id": "", + │ "type": "function" + │ } + │ ] + │ }, + │ { + │ "content": [ + │ { + │ "text": "pi_tool_ok", + │ "type": "text" + │ } + │ ], + │ "role": "tool", + │ "tool_call_id": "tool-call-1" + │ } + │ ] + │ output: [ + │ { + │ "finish_reason": "stop", + │ "index": 0, + │ "message": { + │ "content": "PI_CODING_AGENT_OK pi_tool_ok", + │ "role": "assistant" + │ } + │ } + │ ] + │ metadata: { + │ "model": "claude-haiku-4-5", + │ "pi_coding_agent.api": "anthropic-messages", + │ "pi_coding_agent.model": "claude-haiku-4-5", + │ "pi_coding_agent.operation": "agent.streamFunction", + │ "pi_coding_agent.stop_reason": "stop", + │ "provider": "anthropic" + │ } + │ metrics: { + │ "completion_tokens": 3, + │ "duration": 0, + │ "prompt_cache_creation_tokens": 0, + │ "prompt_cached_tokens": 0, + │ "prompt_tokens": 5, + │ "time_to_first_token": 0, + │ "tokens": 8 + │ } + └── bash [tool] + input: { + "command": "printf pi_tool_ok" + } + output: { + "content": [ + { + "text": "pi_tool_ok", + "type": "text" + } + ] + } + metadata: { + "gen_ai.tool.call.id": "tool-call-1", + "gen_ai.tool.name": "bash", + "pi_coding_agent.tool.name": "bash" + } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/package.json b/e2e/scenarios/pi-coding-agent-instrumentation/package.json index b7b12af73..966f2b802 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/package.json +++ b/e2e/scenarios/pi-coding-agent-instrumentation/package.json @@ -6,17 +6,24 @@ "dependencies": { "pi-coding-agent-v0-latest": { "package": "@earendil-works/pi-coding-agent", - "range": ">=0.0.0 <0.80.0" + "range": "0" + }, + "pi-coding-agent-v081-latest": { + "package": "@earendil-works/pi-coding-agent", + "range": ">=0.81.0 <0.82.0" } } } }, "dependencies": { "pi-coding-agent-v0": "npm:@earendil-works/pi-coding-agent@0.79.1", - "pi-coding-agent-v0-latest": "npm:@earendil-works/pi-coding-agent@0.79.10" + "pi-coding-agent-v0-latest": "npm:@earendil-works/pi-coding-agent@0.84.3", + "pi-coding-agent-v081": "npm:@earendil-works/pi-coding-agent@0.81.0", + "pi-coding-agent-v081-latest": "npm:@earendil-works/pi-coding-agent@0.81.1" }, "pnpm": { "overrides": { + "@earendil-works/pi-coding-agent@0.81.0>@earendil-works/pi-agent-core": "0.81.0", "brace-expansion@>=3.0.0 <5.0.9": "^5.0.9", "undici@<8.9.0": "^8.9.0" } diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/pnpm-lock.yaml b/e2e/scenarios/pi-coding-agent-instrumentation/pnpm-lock.yaml index e7732c1b1..2195c22d5 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/pnpm-lock.yaml +++ b/e2e/scenarios/pi-coding-agent-instrumentation/pnpm-lock.yaml @@ -5,6 +5,7 @@ settings: excludeLinksFromLockfile: false overrides: + '@earendil-works/pi-coding-agent@0.81.0>@earendil-works/pi-agent-core': 0.81.0 brace-expansion@>=3.0.0 <5.0.9: ^5.0.9 undici@<8.9.0: ^8.9.0 @@ -16,8 +17,14 @@ importers: specifier: npm:@earendil-works/pi-coding-agent@0.79.1 version: '@earendil-works/pi-coding-agent@0.79.1(ws@8.21.0)(zod@4.4.3)' pi-coding-agent-v0-latest: - specifier: npm:@earendil-works/pi-coding-agent@0.79.10 - version: '@earendil-works/pi-coding-agent@0.79.10(ws@8.21.0)(zod@4.4.3)' + specifier: npm:@earendil-works/pi-coding-agent@0.84.3 + version: '@earendil-works/pi-coding-agent@0.84.3(ws@8.21.0)(zod@4.4.3)' + pi-coding-agent-v081: + specifier: npm:@earendil-works/pi-coding-agent@0.81.0 + version: '@earendil-works/pi-coding-agent@0.81.0(ws@8.21.0)(zod@4.4.3)' + pi-coding-agent-v081-latest: + specifier: npm:@earendil-works/pi-coding-agent@0.81.1 + version: '@earendil-works/pi-coding-agent@0.81.1(ws@8.21.0)(zod@4.4.3)' packages: @@ -135,14 +142,22 @@ packages: resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} - '@earendil-works/pi-agent-core@0.79.10': - resolution: {integrity: sha512-XKxgdjhcPuyjrthCOFSgfzT3xZ1uBrJ1IMVDxci1to6hIN6BIg9J5iY8q0pGXK1DLgATLP23da+1UyZLwA360Q==} - engines: {node: '>=22.19.0'} - '@earendil-works/pi-agent-core@0.79.4': resolution: {integrity: sha512-xkaZ3yK2XbP9HYdHrrdj/6HqZPM0o/mwbjMSU4RTJyR3HjDG0ZrPz76Hg6s0W+G4u6PpJr1mGx/srCG+3eQA8A==} engines: {node: '>=22.19.0'} + '@earendil-works/pi-agent-core@0.81.0': + resolution: {integrity: sha512-Fe6JYbW0CGVvmD4dwuFreynwZxlTElIMFRqYDl4llRiwfYZjjtAqbV1hY8MrdGqP15+FySBv5RGodOsHqlhvZQ==} + engines: {node: '>=22.19.0'} + + '@earendil-works/pi-agent-core@0.81.1': + resolution: {integrity: sha512-yqbh68CyhqxMov/jUogFJfMqlu2Gd37GAki+tr59YCmAPHfomiCA5ESzusXtpGzABeiZFC/OrRdQ4GwCCOMIHA==} + engines: {node: '>=22.19.0'} + + '@earendil-works/pi-agent-core@0.84.3': + resolution: {integrity: sha512-VURr+xBRl3RxYcw3kT9Pn3yfi6LbRoCJgHF7h1mAblMjtLNV/MfG/RyF0uJizBAM886AEakSiw3j9c/aSngppg==} + engines: {node: '>=22.19.0'} + '@earendil-works/pi-ai@0.79.10': resolution: {integrity: sha512-9jR23tOl0BIUdQMn70Gr72xYBpM7Xgl9Lyv7gAnU1USfkNRuYG/f/edLl+n/Dp/RafDW3JI4DF7y/GhgkORuew==} engines: {node: '>=22.19.0'} @@ -153,24 +168,60 @@ packages: engines: {node: '>=22.19.0'} hasBin: true + '@earendil-works/pi-ai@0.81.1': + resolution: {integrity: sha512-hzHE7Z8l5mgJk+ke67Lge0rwS2+wbKJrFKl9o5M1R1rh33+cCT7D1AHz1OAtX5wFs90E1/BTGhyJRTUHaMxGvQ==} + engines: {node: '>=22.19.0'} + hasBin: true + + '@earendil-works/pi-ai@0.84.3': + resolution: {integrity: sha512-M0YUV8vNO3y2WwWSyY8ijKJV5W4gkSUixuvk+Z00ZBjsyMfsdXfITsHEwP1UIf09YRWXT6oGn0GlCamt+P32XQ==} + engines: {node: '>=22.19.0'} + hasBin: true + + '@earendil-works/pi-client@0.84.3': + resolution: {integrity: sha512-zfErYane+390W0xpBJ/FWCp6aktPpkpcIcXUeZiAziWLoxE80ZNQALRyOSa/gGS5V+1OkNnMYxRxbzN0zUvnOA==} + engines: {node: '>=22.19.0'} + '@earendil-works/pi-coding-agent@0.79.1': resolution: {integrity: sha512-dLnje4U5H3/ZytJpvhjhPINeDT/yvx85e4OH/ziMQRLpPlfNP12/peY9jRQd4W11Xth2+y2xGAFwS+NeVf2ZwA==} engines: {node: '>=22.19.0'} hasBin: true - '@earendil-works/pi-coding-agent@0.79.10': - resolution: {integrity: sha512-YxaRhmgyDTvLDdGVbe7YzTHV80oL5mX5odg6EhGHz3w5Wu1Ix8DCw7bhtiOBLGQNFRcknia0zPmVWIj30XP1EA==} + '@earendil-works/pi-coding-agent@0.81.0': + resolution: {integrity: sha512-2p0Dnx+3fkPLga8M82eg14ZYNLcFLhqxxKyVVfqUSIio9Xx4p7UjvJtopx/6PTeJKmdJl1/xOm/c02AFcJ+l/g==} + engines: {node: '>=22.19.0'} + hasBin: true + + '@earendil-works/pi-coding-agent@0.81.1': + resolution: {integrity: sha512-r6ovAsZOgAqbC/aU6s+/dPnv/sGZBuWyZNvi3pXjpbuX5wvp3XvGkQI7/VLvX2o9XpmpFaPUxKNym1WfkN/P8A==} + engines: {node: '>=22.19.0'} + hasBin: true + + '@earendil-works/pi-coding-agent@0.84.3': + resolution: {integrity: sha512-Yr2p9PubrbFZmYEPYI+C8KmZP9xlFuLDnAG64RtU0ZDgrdiXYWa+y7WGyJO5OlqPliOkVCMd9IzVszO3/t0D0w==} engines: {node: '>=22.19.0'} hasBin: true - '@earendil-works/pi-tui@0.79.10': - resolution: {integrity: sha512-FUVOjDn1DVwM1uHD5MNYboXQrXjIDbSt+BQ3py7nQWCY62tKfxgiM1OBMxTcwRWLfSdZHUPpV0hm1loIdUJnPw==} + '@earendil-works/pi-protocol@0.84.3': + resolution: {integrity: sha512-9a4g6WhLOvRqvsIOFaWxg/2gdrbY4Thclwj5ipLUPAWChfsDJ/8XdPc2sRhSOkD6EsxpEFJz3xppcfwI6EcZDg==} + engines: {node: '>=22.19.0'} + + '@earendil-works/pi-telemetry@0.84.3': + resolution: {integrity: sha512-sgEkWoKrvSGaKn+YfLLFZmn+/A7B/w62eLwTD57nI+C9to8ITlFFVbgC2OtwvPnT3NFGHdCd53qhBEMIlptD1g==} engines: {node: '>=22.19.0'} '@earendil-works/pi-tui@0.79.4': resolution: {integrity: sha512-/ZhfFiHSBMH7AbDrBQIN+UWlJnl9tSEpLYICRGGMzmNfyCqX+30NYacIhyOEaD8R5rS6wJZysAOPU0yNwigbXw==} engines: {node: '>=22.19.0'} + '@earendil-works/pi-tui@0.81.1': + resolution: {integrity: sha512-OMEe+Zt8oQYi/rCq3upxsTlIScWL0FPhXwQus34TbQb3EmTx88S7Uzx32JxvQiEeWOw8eDCdJf2PBUBE9r6wIg==} + engines: {node: '>=22.19.0'} + + '@earendil-works/pi-tui@0.84.3': + resolution: {integrity: sha512-fS6OEQKEEALnKa6Uw8LcgZZ+9CWck7f3MQSCETQp6leUgIFwMEDtKmOUnL9nsYm+RIPmy7OmplVxYRbV6hiaFg==} + engines: {node: '>=22.19.0'} + '@google/genai@1.52.0': resolution: {integrity: sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==} engines: {node: '>=20.0.0'} @@ -446,6 +497,10 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + grok-mermaid@0.2.2: + resolution: {integrity: sha512-XcJEP5dDC8liHBh52mlLjU18fNvu1ckFsu0QpIG3+APZ270fsj9wxpiA6cOURmbUEuoMVgjbC2+UYgTdCqqgzA==} + engines: {node: '>=18'} + highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} @@ -534,6 +589,17 @@ packages: zod: optional: true + openai@6.40.0: + resolution: {integrity: sha512-MWtTjd/gQt4jpbji61NTgFWJLoY/PdRJ6wG9/ZDRMYNMlBKrCrSlkLI+KgHP1vR1qT6LKSAyAqIxno6lcK9JiA==} + peerDependencies: + ws: ^8.18.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + p-retry@4.6.2: resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} engines: {node: '>=8'} @@ -599,6 +665,9 @@ packages: typebox@1.1.38: resolution: {integrity: sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==} + typebox@1.3.7: + resolution: {integrity: sha512-meKuifc33Pccx0O6PdIzYMq3Og8zvP4TIi/a+Bw3AEMZMxOD0+RHGQvpglEe6Zdy3wZ8nqn/j95h8LUZLk/6Hg==} + undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} @@ -606,6 +675,10 @@ packages: resolution: {integrity: sha512-HvltHd7avK13QIw/oLe4qoOLyoVSoafqJ2jYOrtMRBkbYT31eiBQ8O0ehRKZiEZCMEyLFQNIADpgCWC5fALvYQ==} engines: {node: '>=22.19.0'} + undici@8.9.0: + resolution: {integrity: sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==} + engines: {node: '>=22.19.0'} + web-streams-polyfill@3.3.3: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} @@ -877,7 +950,7 @@ snapshots: '@babel/runtime@7.29.7': {} - '@earendil-works/pi-agent-core@0.79.10(ws@8.21.0)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.79.4(ws@8.21.0)(zod@4.4.3)': dependencies: '@earendil-works/pi-ai': 0.79.10(ws@8.21.0)(zod@4.4.3) ignore: 7.0.5 @@ -891,9 +964,23 @@ snapshots: - ws - zod - '@earendil-works/pi-agent-core@0.79.4(ws@8.21.0)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.81.0(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-ai': 0.79.4(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.81.1(ws@8.21.0)(zod@4.4.3) + ignore: 7.0.5 + typebox: 1.1.38 + yaml: 2.9.0 + transitivePeerDependencies: + - '@modelcontextprotocol/sdk' + - bufferutil + - supports-color + - utf-8-validate + - ws + - zod + + '@earendil-works/pi-agent-core@0.81.1(ws@8.21.0)(zod@4.4.3)': + dependencies: + '@earendil-works/pi-ai': 0.81.1(ws@8.21.0)(zod@4.4.3) ignore: 7.0.5 typebox: 1.1.38 yaml: 2.9.0 @@ -905,6 +992,22 @@ snapshots: - ws - zod + '@earendil-works/pi-agent-core@0.84.3(ws@8.21.0)(zod@4.4.3)': + dependencies: + '@earendil-works/pi-ai': 0.84.3(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-telemetry': 0.84.3 + diff: 8.0.4 + ignore: 7.0.5 + typebox: 1.3.7 + yaml: 2.9.0 + transitivePeerDependencies: + - '@modelcontextprotocol/sdk' + - bufferutil + - supports-color + - utf-8-validate + - ws + - zod + '@earendil-works/pi-ai@0.79.10(ws@8.21.0)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) @@ -946,6 +1049,51 @@ snapshots: - ws - zod + '@earendil-works/pi-ai@0.81.1(ws@8.21.0)(zod@4.4.3)': + dependencies: + '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) + '@aws-sdk/client-bedrock-runtime': 3.1048.0 + '@google/genai': 1.52.0 + '@mistralai/mistralai': 2.2.6(@opentelemetry/api@1.9.0) + '@opentelemetry/api': 1.9.0 + '@smithy/node-http-handler': 4.7.3 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + openai: 6.26.0(ws@8.21.0)(zod@4.4.3) + partial-json: 0.1.7 + typebox: 1.1.38 + transitivePeerDependencies: + - '@modelcontextprotocol/sdk' + - bufferutil + - supports-color + - utf-8-validate + - ws + - zod + + '@earendil-works/pi-ai@0.84.3(ws@8.21.0)(zod@4.4.3)': + dependencies: + '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) + '@aws-sdk/client-bedrock-runtime': 3.1048.0 + '@earendil-works/pi-telemetry': 0.84.3 + '@google/genai': 1.52.0 + '@smithy/node-http-handler': 4.7.3 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + openai: 6.40.0(ws@8.21.0)(zod@4.4.3) + partial-json: 0.1.7 + typebox: 1.3.7 + transitivePeerDependencies: + - '@modelcontextprotocol/sdk' + - bufferutil + - supports-color + - utf-8-validate + - ws + - zod + + '@earendil-works/pi-client@0.84.3': + dependencies: + '@earendil-works/pi-protocol': 0.84.3 + '@earendil-works/pi-coding-agent@0.79.1(ws@8.21.0)(zod@4.4.3)': dependencies: '@earendil-works/pi-agent-core': 0.79.4(ws@8.21.0)(zod@4.4.3) @@ -975,11 +1123,11 @@ snapshots: - ws - zod - '@earendil-works/pi-coding-agent@0.79.10(ws@8.21.0)(zod@4.4.3)': + '@earendil-works/pi-coding-agent@0.81.0(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.79.10(ws@8.21.0)(zod@4.4.3) - '@earendil-works/pi-ai': 0.79.10(ws@8.21.0)(zod@4.4.3) - '@earendil-works/pi-tui': 0.79.10 + '@earendil-works/pi-agent-core': 0.81.0(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.81.1(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-tui': 0.81.1 '@silvia-odwyer/photon-node': 0.3.4 chalk: 5.6.2 cross-spawn: 7.0.6 @@ -1005,16 +1153,89 @@ snapshots: - ws - zod - '@earendil-works/pi-tui@0.79.10': + '@earendil-works/pi-coding-agent@0.81.1(ws@8.21.0)(zod@4.4.3)': dependencies: - get-east-asian-width: 1.6.0 - marked: 18.0.5 + '@earendil-works/pi-agent-core': 0.81.1(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.81.1(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-tui': 0.81.1 + '@silvia-odwyer/photon-node': 0.3.4 + chalk: 5.6.2 + cross-spawn: 7.0.6 + diff: 8.0.4 + glob: 13.0.6 + highlight.js: 10.7.3 + hosted-git-info: 9.0.3 + ignore: 7.0.5 + jiti: 2.7.0 + minimatch: 10.2.5 + proper-lockfile: 4.1.2 + semver: 7.8.0 + typebox: 1.1.38 + undici: 8.10.0 + yaml: 2.9.0 + optionalDependencies: + '@mariozechner/clipboard': 0.3.9 + transitivePeerDependencies: + - '@modelcontextprotocol/sdk' + - bufferutil + - supports-color + - utf-8-validate + - ws + - zod + + '@earendil-works/pi-coding-agent@0.84.3(ws@8.21.0)(zod@4.4.3)': + dependencies: + '@earendil-works/pi-agent-core': 0.84.3(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.84.3(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-client': 0.84.3 + '@earendil-works/pi-protocol': 0.84.3 + '@earendil-works/pi-tui': 0.84.3 + '@silvia-odwyer/photon-node': 0.3.4 + chalk: 5.6.2 + cross-spawn: 7.0.6 + diff: 8.0.4 + grok-mermaid: 0.2.2 + highlight.js: 10.7.3 + hosted-git-info: 9.0.3 + ignore: 7.0.5 + jiti: 2.7.0 + minimatch: 10.2.5 + proper-lockfile: 4.1.2 + semver: 7.8.0 + typebox: 1.3.7 + undici: 8.9.0 + yaml: 2.9.0 + optionalDependencies: + '@mariozechner/clipboard': 0.3.9 + transitivePeerDependencies: + - '@modelcontextprotocol/sdk' + - bufferutil + - supports-color + - utf-8-validate + - ws + - zod + + '@earendil-works/pi-protocol@0.84.3': + dependencies: + typebox: 1.3.7 + + '@earendil-works/pi-telemetry@0.84.3': {} '@earendil-works/pi-tui@0.79.4': dependencies: get-east-asian-width: 1.6.0 marked: 15.0.12 + '@earendil-works/pi-tui@0.81.1': + dependencies: + get-east-asian-width: 1.6.0 + marked: 18.0.5 + + '@earendil-works/pi-tui@0.84.3': + dependencies: + get-east-asian-width: 1.6.0 + marked: 18.0.5 + '@google/genai@1.52.0': dependencies: google-auth-library: 10.7.0 @@ -1279,6 +1500,8 @@ snapshots: graceful-fs@4.2.11: {} + grok-mermaid@0.2.2: {} + highlight.js@10.7.3: {} hosted-git-info@9.0.3: @@ -1354,6 +1577,11 @@ snapshots: ws: 8.21.0 zod: 4.4.3 + openai@6.40.0(ws@8.21.0)(zod@4.4.3): + optionalDependencies: + ws: 8.21.0 + zod: 4.4.3 + p-retry@4.6.2: dependencies: '@types/retry': 0.12.0 @@ -1416,10 +1644,14 @@ snapshots: typebox@1.1.38: {} + typebox@1.3.7: {} + undici-types@7.24.6: {} undici@8.10.0: {} + undici@8.9.0: {} + web-streams-polyfill@3.3.3: {} which@2.0.2: diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs index 80a35d9c0..5acb45b5b 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.impl.mjs @@ -9,20 +9,41 @@ export const SCENARIO_NAME = "pi-coding-agent-instrumentation"; async function runPiCodingAgentScenario({ decorateSDK, sdk }) { const instrumentedSDK = decorateSDK ? decorateSDK(sdk) : sdk; + const useFakeStream = process.env.PI_CODING_AGENT_FAKE_STREAM === "true"; const { AuthStorage, DefaultResourceLoader, ModelRegistry, + ModelRuntime, SessionManager, createAgentSession, } = instrumentedSDK; - const authStorage = AuthStorage.inMemory(); - authStorage.setRuntimeApiKey("anthropic", process.env.ANTHROPIC_API_KEY); - const modelRegistry = ModelRegistry.inMemory(authStorage); - modelRegistry.registerProvider("anthropic", { - baseUrl: process.env.ANTHROPIC_BASE_URL, - }); + let authStorage; + let modelRuntime; + let modelRegistry; + if (ModelRuntime) { + modelRuntime = await ModelRuntime.create({ modelsPath: null }); + if (!useFakeStream) { + modelRuntime.registerProvider("anthropic", { + baseUrl: process.env.ANTHROPIC_BASE_URL, + }); + } + await modelRuntime.setRuntimeApiKey( + "anthropic", + useFakeStream + ? "pi-coding-agent-e2e-placeholder" + : process.env.ANTHROPIC_API_KEY, + ); + modelRegistry = new ModelRegistry(modelRuntime); + } else { + authStorage = AuthStorage.inMemory(); + authStorage.setRuntimeApiKey("anthropic", process.env.ANTHROPIC_API_KEY); + modelRegistry = ModelRegistry.inMemory(authStorage); + modelRegistry.registerProvider("anthropic", { + baseUrl: process.env.ANTHROPIC_BASE_URL, + }); + } const model = modelRegistry.find("anthropic", "claude-haiku-4-5"); if (!model) { throw new Error("Expected Pi Coding Agent Anthropic model"); @@ -43,16 +64,62 @@ async function runPiCodingAgentScenario({ decorateSDK, sdk }) { "prompt", async () => { const result = await createAgentSession({ - authStorage, cwd, model, - modelRegistry, + ...(modelRuntime + ? { modelRuntime } + : { authStorage, modelRegistry }), resourceLoader, sessionManager: SessionManager.inMemory(cwd), thinkingLevel: "off", tools: ["bash"], }); session = result.session; + if (useFakeStream) { + let streamCallCount = 0; + session.agent.streamFunction = async (streamModel) => { + streamCallCount += 1; + const message = { + api: "anthropic-messages", + content: + streamCallCount === 1 + ? [ + { + arguments: { command: "printf pi_tool_ok" }, + id: "tool-call-1", + name: "bash", + type: "toolCall", + }, + ] + : [ + { + text: "PI_CODING_AGENT_OK pi_tool_ok", + type: "text", + }, + ], + model: streamModel.id, + provider: streamModel.provider, + role: "assistant", + stopReason: streamCallCount === 1 ? "toolUse" : "stop", + timestamp: Date.now(), + usage: { + cacheRead: 0, + cacheWrite: 0, + input: 5, + output: 3, + totalTokens: 8, + }, + }; + return { + async *[Symbol.asyncIterator]() { + yield { message, type: "done" }; + }, + async result() { + return message; + }, + }; + }; + } await session.prompt( "Use the bash tool to run `printf pi_tool_ok` exactly once, then reply with exactly PI_CODING_AGENT_OK and include the command output.", { expandPromptTemplates: false, source: "rpc" }, diff --git a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.test.ts b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.test.ts index d7d905617..6b6575e1c 100644 --- a/e2e/scenarios/pi-coding-agent-instrumentation/scenario.test.ts +++ b/e2e/scenarios/pi-coding-agent-instrumentation/scenario.test.ts @@ -25,10 +25,29 @@ const piCodingAgentScenarios = await Promise.all( autoEntry: "scenario.pi-coding-agent-v079.mjs", autoSnapshotName: "pi-coding-agent-v0-latest-auto-hook", dependencyName: "pi-coding-agent-v0-latest", + fakeStream: true, variantKey: "pi-coding-agent-v0-latest", wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", wrapperSnapshotName: "pi-coding-agent-v0-latest-wrapped", }, + { + autoEntry: "scenario.pi-coding-agent-v079.mjs", + autoSnapshotName: "pi-coding-agent-v081-auto-hook", + dependencyName: "pi-coding-agent-v081", + fakeStream: true, + variantKey: "pi-coding-agent-v081", + wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", + wrapperSnapshotName: "pi-coding-agent-v081-wrapped", + }, + { + autoEntry: "scenario.pi-coding-agent-v079.mjs", + autoSnapshotName: "pi-coding-agent-v081-latest-auto-hook", + dependencyName: "pi-coding-agent-v081-latest", + fakeStream: true, + variantKey: "pi-coding-agent-v081-latest", + wrapperEntry: "scenario.pi-coding-agent-v079-wrapped.mjs", + wrapperSnapshotName: "pi-coding-agent-v081-latest-wrapped", + }, ].map(async (scenario) => ({ ...scenario, version: await readInstalledPackageVersion( @@ -51,8 +70,14 @@ describe.concurrent("variants", () => { env: { PI_CODING_AGENT_PACKAGE_NAME: piCodingAgentScenario.dependencyName, + ...(piCodingAgentScenario.fakeStream + ? { PI_CODING_AGENT_FAKE_STREAM: "true" } + : {}), }, runContext: { + ...(piCodingAgentScenario.fakeStream + ? { cassette: false as const } + : {}), variantKey: piCodingAgentScenario.variantKey, originalScenarioDir, }, @@ -73,9 +98,15 @@ describe.concurrent("variants", () => { env: { PI_CODING_AGENT_PACKAGE_NAME: piCodingAgentScenario.dependencyName, + ...(piCodingAgentScenario.fakeStream + ? { PI_CODING_AGENT_FAKE_STREAM: "true" } + : {}), }, nodeArgs: ["--import", "braintrust/hook.mjs"], runContext: { + ...(piCodingAgentScenario.fakeStream + ? { cassette: false as const } + : {}), variantKey: piCodingAgentScenario.variantKey, originalScenarioDir, }, diff --git a/js/src/auto-instrumentations/configs/pi-coding-agent.test.ts b/js/src/auto-instrumentations/configs/pi-coding-agent.test.ts index dbc8e0160..4446b7872 100644 --- a/js/src/auto-instrumentations/configs/pi-coding-agent.test.ts +++ b/js/src/auto-instrumentations/configs/pi-coding-agent.test.ts @@ -12,7 +12,7 @@ describe("piCodingAgentConfigs", () => { channelName: piCodingAgentChannelName, module: { name: "@earendil-works/pi-coding-agent", - versionRange: ">=0.79.0 <0.82.0", + versionRange: ">=0.79.0 <1.0.0", filePath: "dist/core/agent-session.js", }, functionQuery: { diff --git a/js/src/auto-instrumentations/configs/pi-coding-agent.ts b/js/src/auto-instrumentations/configs/pi-coding-agent.ts index 0bf6e2a56..40de6927e 100644 --- a/js/src/auto-instrumentations/configs/pi-coding-agent.ts +++ b/js/src/auto-instrumentations/configs/pi-coding-agent.ts @@ -1,7 +1,7 @@ import type { InstrumentationConfig } from "../orchestrion-js"; import { piCodingAgentChannels } from "../../instrumentation/plugins/pi-coding-agent-channels"; -const piCodingAgentVersionRange = ">=0.79.0 <0.82.0"; +const piCodingAgentVersionRange = ">=0.79.0 <1.0.0"; export const piCodingAgentConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts b/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts index 433361974..0da3cf23f 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts +++ b/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts @@ -91,113 +91,120 @@ describe("PiCodingAgentPlugin", () => { expect(mockRemoveInterceptor).toHaveBeenCalledTimes(1); }); - it("uses interceptor ALS for prompt, LLM, and tool spans", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); - const finalMessage = makeAssistantMessage("done"); - const originalStreamFn = vi.fn(async () => { - expect(isAutoInstrumentationSuppressed()).toBe(true); - return makeStream(finalMessage); - }); - const agent = makeAgent(originalStreamFn); - agent.state.tools = [bashTool()]; - const session = makeSession(agent); - const context = { - systemPrompt: "system", - messages: [ - { - role: "user", - content: [{ type: "text", text: "hello" }], - timestamp: 1, - }, - ], - tools: [bashTool()], - }; - - await interceptor( - async function (this: typeof session) { + it.each([ + { property: "streamFn" as const, version: "0.79.1" }, + { property: "streamFunction" as const, version: "0.81.0" }, + ])( + "uses interceptor ALS for prompt, LLM, and tool spans with $property", + async ({ property, version }) => { + const interceptor = promptInterceptor(enablePlugin(plugins)); + const finalMessage = makeAssistantMessage("done"); + const originalStreamFn = vi.fn(async () => { expect(isAutoInstrumentationSuppressed()).toBe(true); - const stream = await this.agent.streamFn(anthropicModel(), context, { - apiKey: "secret", - headers: { authorization: "secret" }, - reasoning: "low", - }); - await this.agent.emit({ - args: { command: "printf pi_tool_ok" }, - toolCallId: "tool-1", - toolName: "bash", - type: "tool_execution_start", - }); - await this.agent.emit({ - isError: false, - result: { stdout: "pi_tool_ok" }, - toolCallId: "tool-1", - toolName: "bash", - type: "tool_execution_end", - }); - await stream.result(); - await this.agent.emit({ - message: finalMessage, - toolResults: [], - turnIndex: 0, - type: "turn_end", - }); - }, - session, - ["hello", undefined], - { moduleVersion: "0.79.1" }, - ); - - expect(isAutoInstrumentationSuppressed()).toBe(false); - expect(agent.streamFn).not.toBe(originalStreamFn); - expect(originalStreamFn).toHaveBeenCalledWith( - anthropicModel(), - context, - expect.objectContaining({ apiKey: "secret" }), - ); + return makeStream(finalMessage); + }); + const agent = makeAgent(originalStreamFn, property); + agent.state.tools = [bashTool()]; + const session = makeSession(agent); + const context = { + systemPrompt: "system", + messages: [ + { + role: "user", + content: [{ type: "text", text: "hello" }], + timestamp: 1, + }, + ], + tools: [bashTool()], + }; - const taskSpan = findSpan(spans, "AgentSession.prompt"); - const llmSpan = findSpan(spans, "anthropic.messages.create"); - const toolSpan = findSpan(spans, "bash"); - expect(llmSpan?.args.event.input).toEqual([ - { role: "system", content: "system" }, - { role: "user", content: [{ type: "text", text: "hello" }] }, - ]); - expect(llmSpan?.args.event.metadata).toMatchObject({ - model: "claude-haiku-4-5", - provider: "anthropic", - tools: [ - { - type: "function", - function: expect.objectContaining({ name: "bash" }), + await interceptor( + async function (this: typeof session) { + expect(isAutoInstrumentationSuppressed()).toBe(true); + const stream = await this.agent[property](anthropicModel(), context, { + apiKey: "secret", + headers: { authorization: "secret" }, + reasoning: "low", + }); + await this.agent.emit({ + args: { command: "printf pi_tool_ok" }, + toolCallId: "tool-1", + toolName: "bash", + type: "tool_execution_start", + }); + await this.agent.emit({ + isError: false, + result: { stdout: "pi_tool_ok" }, + toolCallId: "tool-1", + toolName: "bash", + type: "tool_execution_end", + }); + await stream.result(); + await this.agent.emit({ + message: finalMessage, + toolResults: [], + turnIndex: 0, + type: "turn_end", + }); }, - ], - }); - expect(llmSpan?.args.event.metadata).not.toHaveProperty("apiKey"); - expect(llmSpan?.args.event.metadata).not.toHaveProperty("headers"); - expect(llmSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ - metrics: expect.objectContaining({ - completion_tokens: 3, - prompt_tokens: 5, - tokens: 8, + session, + ["hello", undefined], + { moduleVersion: version }, + ); + + expect(isAutoInstrumentationSuppressed()).toBe(false); + expect(agent[property]).not.toBe(originalStreamFn); + expect(originalStreamFn).toHaveBeenCalledWith( + anthropicModel(), + context, + expect.objectContaining({ apiKey: "secret" }), + ); + + const taskSpan = findSpan(spans, "AgentSession.prompt"); + const llmSpan = findSpan(spans, "anthropic.messages.create"); + const toolSpan = findSpan(spans, "bash"); + expect(llmSpan?.args.event.input).toEqual([ + { role: "system", content: "system" }, + { role: "user", content: [{ type: "text", text: "hello" }] }, + ]); + expect(llmSpan?.args.event.metadata).toMatchObject({ + model: "claude-haiku-4-5", + "pi_coding_agent.operation": `agent.${property}`, + provider: "anthropic", + tools: [ + { + type: "function", + function: expect.objectContaining({ name: "bash" }), + }, + ], + }); + expect(llmSpan?.args.event.metadata).not.toHaveProperty("apiKey"); + expect(llmSpan?.args.event.metadata).not.toHaveProperty("headers"); + expect(llmSpan?.log).toHaveBeenCalledWith( + expect.objectContaining({ + metrics: expect.objectContaining({ + completion_tokens: 3, + prompt_tokens: 5, + tokens: 8, + }), + output: expect.any(Array), }), - output: expect.any(Array), - }), - ); - expect(toolSpan?.args.event.input).toEqual({ - command: "printf pi_tool_ok", - }); - expect(toolSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ output: { stdout: "pi_tool_ok" } }), - ); - const finalTaskLog = taskSpan?.log.mock.calls.at(-1)?.[0]; - expect(finalTaskLog?.metrics).toMatchObject({ - completion_tokens: 3, - prompt_tokens: 5, - tokens: 8, - }); - expect(taskSpan?.end).toHaveBeenCalledTimes(1); - }); + ); + expect(toolSpan?.args.event.input).toEqual({ + command: "printf pi_tool_ok", + }); + expect(toolSpan?.log).toHaveBeenCalledWith( + expect.objectContaining({ output: { stdout: "pi_tool_ok" } }), + ); + const finalTaskLog = taskSpan?.log.mock.calls.at(-1)?.[0]; + expect(finalTaskLog?.metrics).toMatchObject({ + completion_tokens: 3, + prompt_tokens: 5, + tokens: 8, + }); + expect(taskSpan?.end).toHaveBeenCalledTimes(1); + }, + ); it("isolates overlapping prompts on the same agent without prompt matching", async () => { const interceptor = promptInterceptor(enablePlugin(plugins)); @@ -521,11 +528,14 @@ function makeIteratorBackedStream(events: any[]) { }; } -function makeAgent(streamFn: any) { +function makeAgent( + streamFunction: any, + property: "streamFn" | "streamFunction" = "streamFn", +): any { const listeners = new Set(); return { state: { model: anthropicModel(), tools: [] as any[] }, - streamFn, + [property]: streamFunction, subscribe: vi.fn((listener) => { listeners.add(listener); return vi.fn(() => listeners.delete(listener)); diff --git a/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts b/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts index da6c50b30..c79f00170 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts +++ b/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts @@ -29,7 +29,7 @@ import type { PiModel, PiPromptOptions, PiSimpleStreamOptions, - PiStreamFn, + PiStreamFunction, PiTextContent, PiTool, PiToolCall, @@ -67,8 +67,9 @@ type PiToolSpanState = { }; type PiAgentPatchState = { - originalStreamFn: PiStreamFn; - wrappedStreamFn: PiStreamFn; + originalStreamFunction: PiStreamFunction; + property: "streamFn" | "streamFunction"; + wrappedStreamFunction: PiStreamFunction; }; const piAgentPatchStates = new WeakMap(); @@ -228,7 +229,8 @@ function extractSession( function isPiAgent(value: unknown): value is PiAgent { return ( isObject(value) && - typeof value.streamFn === "function" && + (typeof value.streamFunction === "function" || + typeof value.streamFn === "function") && typeof value.subscribe === "function" ); } @@ -245,17 +247,27 @@ function currentPiPromptState(): PiPromptState | undefined { } function installPiAgentInstrumentation(agent: PiAgent): void { + const property = + typeof agent.streamFunction === "function" ? "streamFunction" : "streamFn"; + const streamFunction = agent[property]; const existing = piAgentPatchStates.get(agent); - if (!existing || agent.streamFn !== existing.wrappedStreamFn) { + if ( + streamFunction && + (!existing || + existing.property !== property || + streamFunction !== existing.wrappedStreamFunction) + ) { const patchState = { - originalStreamFn: agent.streamFn, - wrappedStreamFn: agent.streamFn, + originalStreamFunction: streamFunction, + property, + wrappedStreamFunction: streamFunction, } satisfies PiAgentPatchState; - patchState.wrappedStreamFn = makeInstrumentedStreamFn( + patchState.wrappedStreamFunction = makeInstrumentedStreamFunction( agent, - patchState.originalStreamFn, + patchState.originalStreamFunction, + property, ); - agent.streamFn = patchState.wrappedStreamFn; + agent[property] = patchState.wrappedStreamFunction; piAgentPatchStates.set(agent, patchState); } @@ -282,24 +294,31 @@ function installPiAgentInstrumentation(agent: PiAgent): void { } } -function makeInstrumentedStreamFn( +function makeInstrumentedStreamFunction( agent: PiAgent, - originalStreamFn: PiStreamFn, -): PiStreamFn { - return async function instrumentedPiStreamFn( + originalStreamFunction: PiStreamFunction, + property: PiAgentPatchState["property"], +): PiStreamFunction { + return async function instrumentedPiStreamFunction( this: unknown, model: PiModel, context: PiContext, options?: PiSimpleStreamOptions, ) { const invokeOriginal = () => - Reflect.apply(originalStreamFn, this, [model, context, options]); + Reflect.apply(originalStreamFunction, this, [model, context, options]); const state = currentPiPromptState(); if (!state || state.agent !== agent || state.finalized) { return invokeOriginal(); } - const llmState = await startPiLlmSpan(state, model, context, options); + const llmState = await startPiLlmSpan( + state, + model, + context, + property, + options, + ); try { const stream = await runWithAutoInstrumentationSuppressed(invokeOriginal); return patchAssistantMessageStream(stream, state, llmState); @@ -314,13 +333,14 @@ async function startPiLlmSpan( state: PiPromptState, model: PiModel, context: PiContext, + property: PiAgentPatchState["property"], options?: PiSimpleStreamOptions, ): Promise { const metadata = { ...extractModelMetadata(model), ...extractStreamOptionsMetadata(options), ...extractToolMetadata(context.tools), - "pi_coding_agent.operation": "agent.streamFn", + "pi_coding_agent.operation": `agent.${property}`, }; const span = startBaseSpan( withSpanInstrumentationName( diff --git a/js/src/vendor-sdk-types/pi-coding-agent.ts b/js/src/vendor-sdk-types/pi-coding-agent.ts index 13df2a1b9..066f17345 100644 --- a/js/src/vendor-sdk-types/pi-coding-agent.ts +++ b/js/src/vendor-sdk-types/pi-coding-agent.ts @@ -39,7 +39,9 @@ export interface PiPromptOptions { } export interface PiAgent { - streamFn: PiStreamFn; + /** @deprecated Renamed to streamFunction in pi-agent-core 0.81.0. */ + streamFn?: PiStreamFunction; + streamFunction?: PiStreamFunction; subscribe(listener: PiAgentEventListener): () => void; readonly state?: { model?: PiModel; @@ -94,7 +96,7 @@ export type PiAgentEvent = isError: boolean; }; -export type PiStreamFn = ( +export type PiStreamFunction = ( model: PiModel, context: PiContext, options?: PiSimpleStreamOptions,