From 5101d082843e08cb2314cc7f017161c0120961d4 Mon Sep 17 00:00:00 2001 From: Ali Ibrahim Jr <48456829+IBJunior@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:40:53 +0200 Subject: [PATCH 1/5] feat(tracing): own the traces with OpenTelemetry instead of Langfuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cameron inherited Langfuse tracing from the starter: a vendor callback pointed at a hosted backend by default. A trace carries prompts, tool arguments and transactions, so that quietly broke the rule that financial data stays on infrastructure the owner controls. - GenAiTracer: a LangChain callback that emits OTel GenAI spans (invoke_agent, chat, execute_tool), records the approval gate as an event (tool names only) and a resume's approve/reject counts. Attribute names live in semconv.ts alone. - Tracing is off unless OTEL_EXPORTER_OTLP_ENDPOINT is set; content capture is a separate opt-in (OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT). Off, no prompt, message, tool argument/result or error message leaves the process. - compose: Phoenix as the default backend, split like the data — dev `phoenix` (--profile tracing, 6006) and prod `phoenix-prod` (--profile full, 6016, own volume), loopback-only, telemetry off. Prod vars are PROD_OTEL_* so a dev value can't redirect them. - Removes @langfuse/*; any OTLP backend (Langfuse, Jaeger, ...) is an endpoint change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .env.eval.example | 2 - .env.example | 15 +- AGENTS.md | 40 ++- README.md | 2 +- compose.yaml | 56 +++- docs/OBSERVABILITY.md | 225 +++++++++------- instrumentation.ts | 27 +- package.json | 5 +- pnpm-lock.yaml | 65 +---- src/lib/tracing/config.test.ts | 34 +++ src/lib/tracing/config.ts | 12 + src/lib/tracing/genAiTracer.test.ts | 405 ++++++++++++++++++++++++++++ src/lib/tracing/genAiTracer.ts | 303 +++++++++++++++++++++ src/lib/tracing/messages.ts | 60 +++++ src/lib/tracing/semconv.ts | 52 ++++ src/services/agentService.ts | 8 +- 16 files changed, 1099 insertions(+), 212 deletions(-) create mode 100644 src/lib/tracing/config.test.ts create mode 100644 src/lib/tracing/config.ts create mode 100644 src/lib/tracing/genAiTracer.test.ts create mode 100644 src/lib/tracing/genAiTracer.ts create mode 100644 src/lib/tracing/messages.ts create mode 100644 src/lib/tracing/semconv.ts diff --git a/.env.eval.example b/.env.eval.example index d225bad..077b67d 100644 --- a/.env.eval.example +++ b/.env.eval.example @@ -16,7 +16,5 @@ S3_BUCKET_NAME=uploads S3_REGION=us-east-1 S3_FORCE_PATH_STYLE=true -LANGFUSE_ENABLED=false - # Evals call a real model — this is the only cost of a run. ANTHROPIC_API_KEY= diff --git a/.env.example b/.env.example index ec2aae7..84b0573 100644 --- a/.env.example +++ b/.env.example @@ -82,12 +82,9 @@ S3_FORCE_PATH_STYLE=true # Required for MinIO, false for AWS S3 # S3_FORCE_PATH_STYLE=false -# Langfuse Observability (optional) -# Set LANGFUSE_ENABLED=true and provide your API keys to enable LLM tracing. -# Cloud: https://cloud.langfuse.com -# Self-hosted: point LANGFUSE_BASE_URL to your own instance (e.g. http://localhost:3030) -# See docs/OBSERVABILITY.md for setup instructions. -LANGFUSE_ENABLED=false -LANGFUSE_SECRET_KEY="sk-lf-..." -LANGFUSE_PUBLIC_KEY="pk-lf-..." -LANGFUSE_BASE_URL="https://cloud.langfuse.com" +# Tracing (optional, OpenTelemetry). Off unless an endpoint is set — nothing leaves the process +# by default. `docker compose --profile tracing up -d` starts a local Phoenix on 6006. +# See docs/OBSERVABILITY.md (other backends, auth headers, what a trace contains). +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:6006 +# Prompts, messages, tool arguments and results are NOT sent unless this is "true". +# OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=false diff --git a/AGENTS.md b/AGENTS.md index cf8dc57..b77eb2c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,12 +42,13 @@ pnpm db:studio # Database UI `compose.yaml` defines **two stacks that share no data**. The separation is physical — separate services, separate named volumes — not a naming convention. -| | Default (`up -d`) | `--profile full` | -| -------- | ---------------------------- | --------------------------------- | -| Postgres | `db` → `mydb_dev`, port 5544 | `db-prod` → `mydb`, port **5546** | -| MinIO | `minio`, 9100/9101 | `minio-prod`, **9102/9103** | -| Volumes | `db_data`, `minio_data` | `prod_db_data`, `prod_minio_data` | -| Contents | seed/scratch data | **real financial records** | +| | Default (`up -d`) | `--profile full` | +| -------- | ----------------------------------------------- | ------------------------------------------------------ | +| Postgres | `db` → `mydb_dev`, port 5544 | `db-prod` → `mydb`, port **5546** | +| MinIO | `minio`, 9100/9101 | `minio-prod`, **9102/9103** | +| Phoenix | `phoenix`, 6006 (only with `--profile tracing`) | `phoenix-prod`, **6016** | +| Volumes | `db_data`, `minio_data`, `phoenix_data` | `prod_db_data`, `prod_minio_data`, `prod_phoenix_data` | +| Contents | seed/scratch data | **real financial records and their traces** | **`docker compose down -v` deletes named volumes.** Because the volume sets are disjoint, a `-v` teardown of the dev stack cannot reach production. Removing production data requires naming that @@ -578,14 +579,25 @@ how this repo's code gets written. Tool conventions here follow npx skills add agentailor/skills --skill tool-design ``` -## Langfuse Observability - -LLM tracing is implemented via two mechanisms — see [docs/OBSERVABILITY.md](docs/OBSERVABILITY.md) for full setup instructions. - -- **`instrumentation.ts`** (project root): Next.js calls `register()` once before any route; initializes the OTel SDK with `LangfuseSpanProcessor` when `LANGFUSE_ENABLED=true` -- **`src/services/agentService.ts`**: Conditionally attaches `CallbackHandler` to each `agent.stream()` call for LangGraph-specific traces (LLM calls, tool invocations, token counts) -- Toggle tracing: set `LANGFUSE_ENABLED=true/false` in `.env` — no code changes needed -- Works with Langfuse Cloud or a self-hosted Docker instance +## Tracing (OpenTelemetry) + +Agent runs are traced as OTel GenAI spans and exported over OTLP to a backend the owner runs — +Phoenix by default. Full detail in [docs/OBSERVABILITY.md](docs/OBSERVABILITY.md). + +- **Off unless `OTEL_EXPORTER_OTLP_ENDPOINT` is set** (`src/lib/tracing/config.ts`). There is no + default destination in code; the production compose stack points at its own `phoenix-prod`. +- **Content capture is a separate opt-in** (`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`). + Off, no prompt, message, tool argument, tool result or error message leaves the process — only + names, counts, timings and token usage. +- **`GenAiTracer`** (`src/lib/tracing/genAiTracer.ts`) is a LangChain callback attached per + request in `agentService.ts`. It spans the run (`invoke_agent`), each model call (`chat`) and + each tool call (`execute_tool`) — never graph nodes or middleware — and records the approval + gate as an event (tool names only) and the resume's approve/reject counts. +- **`semconv.ts` is the only file naming `gen_ai.*` / `cameron.*` attributes.** The conventions are + still Development stability; isolate any rename there. +- **`instrumentation.ts`** starts the Node SDK with the OTLP/proto exporter. Next.js adds its own + route and `fetch` spans to the same trace. +- No vendor SDK: Langfuse, Jaeger or anything OTLP is an endpoint change, not a code change. ## OpenAPI / API Docs diff --git a/README.md b/README.md index b4cf2f1..66ae9b7 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ The technical detail lives in [`docs/`](docs/) rather than here: | [TESTING.md](docs/TESTING.md) | Why the unit suite stays free and offline, and what belongs in evals instead | | [API.md](docs/API.md) | The generated OpenAPI spec, served at `/api/openapi` and browsable at `/api-docs` | | [OAUTH.md](docs/OAUTH.md) | OAuth for MCP servers that need it | -| [OBSERVABILITY.md](docs/OBSERVABILITY.md) | Langfuse tracing setup | +| [OBSERVABILITY.md](docs/OBSERVABILITY.md) | OpenTelemetry tracing to a self-hosted backend (Phoenix by default), what a trace contains, and swapping backends | | [eval/README.md](eval/README.md) | The paid, non-deterministic eval suite — never in CI | [AGENTS.md](AGENTS.md) carries the conventions a coding agent needs to work in this repo. diff --git a/compose.yaml b/compose.yaml index 0531cbb..d5c8731 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,13 +2,15 @@ # # docker compose up -d DEV dependencies: Postgres `mydb_dev` (5544) + MinIO # (9100/9101), app on the host. Disposable by design. -# docker compose --profile full up PRODUCTION: the whole app on its OWN Postgres (5546) and -# its OWN MinIO (9102/9103). This is the database holding -# real financial records. +# docker compose --profile tracing up -d +# ...plus the DEV trace backend, Phoenix (6006). +# docker compose --profile full up PRODUCTION: the whole app on its OWN Postgres (5546), its +# OWN MinIO (9102/9103) and its OWN Phoenix (6016). This is +# the stack holding real financial records — and their traces. # -# The split is physical, not by convention: `db-prod` and `minio-prod` use their own named volumes, -# so `docker compose down -v` (which deletes volumes) on the default profile CANNOT reach -# production data. Removing production data takes naming that stack explicitly. +# The split is physical, not by convention: `db-prod`, `minio-prod` and `phoenix-prod` use their +# own named volumes, so `docker compose down -v` (which deletes volumes) on the default profile +# CANNOT reach production data. Removing production data takes naming that stack explicitly. # # docker compose down stop dev, keep data # docker compose down -v stop dev, DESTROY dev data (safe: dev only) @@ -74,6 +76,21 @@ services: exit 0; " + # Dev trace backend, `--profile tracing` only. Point the host app at it with + # OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:6006 (see docs/OBSERVABILITY.md). + phoenix: + profiles: [tracing] + image: arizephoenix/phoenix:version-20.23.0-nonroot + container_name: cameron-ai-phoenix + environment: + PHOENIX_TELEMETRY_ENABLED: "false" + ports: + # Loopback only: Phoenix has no login, and traces describe your finances. + - "127.0.0.1:${PHOENIX_PORT:-6006}:6006" + volumes: + # The nonroot image can only write under its own home, so the volume mounts there. + - phoenix_data:/home/nonroot + # ── `--profile full` only: PRODUCTION. Real financial data. ──────────────────────────────────── # Separate service and separate volume from `db` above — that separation is the whole point, so @@ -135,6 +152,19 @@ services: exit 0; " + # Production traces: the same split as the database, so a dev teardown cannot reach them. + phoenix-prod: + profiles: [full] + image: arizephoenix/phoenix:version-20.23.0-nonroot + container_name: cameron-ai-phoenix-prod + environment: + PHOENIX_TELEMETRY_ENABLED: "false" + ports: + - "127.0.0.1:${PHOENIX_PROD_PORT:-6016}:6006" + volumes: + - prod_phoenix_data:/home/nonroot + restart: unless-stopped + # Applies pending migrations, then exits. Its own service (and its own build stage) because # drizzle-kit is a devDependency that the runtime image deliberately does not carry. migrate: @@ -183,16 +213,20 @@ services: OPENAI_API_KEY: ${OPENAI_API_KEY:-} GOOGLE_API_KEY: ${GOOGLE_API_KEY:-} - LANGFUSE_ENABLED: ${LANGFUSE_ENABLED:-false} - LANGFUSE_SECRET_KEY: ${LANGFUSE_SECRET_KEY:-} - LANGFUSE_PUBLIC_KEY: ${LANGFUSE_PUBLIC_KEY:-} - LANGFUSE_BASE_URL: ${LANGFUSE_BASE_URL:-https://cloud.langfuse.com} + # Traces go to this stack's own Phoenix, metadata only. PROD_-prefixed so a host-dev value in + # .env (e.g. localhost:6006) never redirects production traces. See docs/OBSERVABILITY.md. + OTEL_EXPORTER_OTLP_ENDPOINT: ${PROD_OTEL_EXPORTER_OTLP_ENDPOINT:-http://phoenix-prod:6006} + OTEL_EXPORTER_OTLP_HEADERS: ${PROD_OTEL_EXPORTER_OTLP_HEADERS:-} + OTEL_SDK_DISABLED: ${PROD_OTEL_SDK_DISABLED:-false} + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: ${PROD_OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT:-false} restart: unless-stopped volumes: # Dev — disposable. db_data: minio_data: - # Production — real financial records. Only a `--profile full down -v` removes these. + phoenix_data: + # Production — real financial records and their traces. Only a `--profile full down -v` removes these. prod_db_data: prod_minio_data: + prod_phoenix_data: diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index d0a250e..98ac3cf 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -1,126 +1,153 @@ -# Observability with Langfuse +# Observability -This document explains how LLM observability is implemented in this template using [Langfuse](https://langfuse.com), and how to configure it for cloud or self-hosted setups. +Cameron traces every agent run with [OpenTelemetry](https://opentelemetry.io), using the +[GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/), and exports the +spans over OTLP to a backend **you** run. The default is [Phoenix](https://phoenix.arize.com), +self-hosted next to Cameron. Any OTLP backend works by changing one variable. -## What is Langfuse? +## Why the traces stay with you -Langfuse is an open-source LLM observability platform. It captures traces of your agent runs, giving you visibility into: +A trace of an agent run can hold the most sensitive data in the system: what you asked, every +transaction a tool read, every amount the model wrote. Cameron's third rule is that financial data +stays on infrastructure you control, and that has to include the traces. So: -- Every LLM call (model, prompt, completion, token usage, latency) -- Tool invocations and their inputs/outputs -- The full agent step sequence per conversation turn -- Costs and error rates over time +- **Tracing is off unless you name an endpoint.** There is no default destination in the code. +- **Content is a second, separate opt-in.** With it off (the default), only metadata leaves the + process — see [What a trace contains](#what-a-trace-contains). +- **There is no vendor SDK.** Cameron emits standard spans; the backend is your choice. -For a `createAgent` agent like this one, a single user message typically produces a trace that shows the model node → tools node chain, with nested LLM calls inside. When a mutating tool is proposed, the `humanInTheLoopMiddleware` pauses the run (an interrupt) until the user approves or denies. +## Quick start (development) -## Integration Architecture - -Two mechanisms work together to provide full coverage: - -``` -Next.js process starts - └─ instrumentation.ts register() called once (before any route) - └─ (if LANGFUSE_ENABLED=true) NodeSDK.start() — OTel active globally - -Request → /api/agent/stream - └─ agentService.ts streamResponse() - └─ agent.stream({ callbacks: [langfuseHandler] }) - ├─ LangGraph events → CallbackHandler → Langfuse - └─ OTel spans (HTTP calls, etc.) → LangfuseSpanProcessor → Langfuse +```bash +docker compose --profile tracing up -d # dev Postgres + MinIO + Phoenix on 127.0.0.1:6006 ``` -**1. OpenTelemetry span processor** (`instrumentation.ts`) - -- Initialized once at process startup via Next.js's built-in instrumentation hook -- Captures infrastructure-level spans: HTTP requests, database calls, anything OTel-instrumented -- Uses `@langfuse/otel` → `LangfuseSpanProcessor` - -**2. LangChain CallbackHandler** (`src/services/agentService.ts`) - -- Attached to each `agent.stream()` call -- Captures LangChain/LangGraph-specific semantic events: chain starts/ends, LLM calls with token counts, tool invocations, agent steps -- Uses `@langfuse/langchain` → `CallbackHandler` - -Both are gated behind `LANGFUSE_ENABLED=true` — when disabled, there is zero overhead and no network calls to Langfuse. - -## Setup - -### Option A: Langfuse Cloud - -1. Create a free account at [cloud.langfuse.com](https://cloud.langfuse.com) (EU region) or [us.cloud.langfuse.com](https://us.cloud.langfuse.com) (US region) -2. Create a project and navigate to **Settings → API Keys** -3. Copy your Secret Key and Public Key -4. Add to your `.env.local`: +Add to `.env.local`, then restart `pnpm dev`: ```bash -LANGFUSE_ENABLED=true -LANGFUSE_SECRET_KEY="sk-lf-..." -LANGFUSE_PUBLIC_KEY="pk-lf-..." -LANGFUSE_BASE_URL="https://cloud.langfuse.com" +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:6006 ``` -5. Restart the dev server — traces will appear in your Langfuse project immediately. +Send a message, then open . Spans land in Phoenix's `default` project; +each Cameron thread is a Phoenix **session**, so an approval pause and its resume sit together. -### Option B: Self-Hosted Langfuse +The trace list shows each trace's top span, which is Next.js's `GET /api/agent/stream` request span, +so its kind reads `unknown`: it carries no GenAI attributes. That is expected — agent and HTTP +tracing share one trace on purpose. Token and cost totals per trace and per session still add up, +because Phoenix sums them over the whole tree; filter spans by `span_kind == 'AGENT'` to list +agent runs alone. -Langfuse ships its own Docker Compose file with all required services (web, worker, Postgres, MinIO, Redis, ClickHouse). The recommended approach is to clone the Langfuse repo and run it directly. +## Production -Full guide: [langfuse.com/self-hosting/deployment/docker-compose](https://langfuse.com/self-hosting/deployment/docker-compose) +`docker compose --profile full up` starts `phoenix-prod` (127.0.0.1:**6016**) with its own volume, +and the `web` service traces to it out of the box — metadata only. Production traces are kept +apart from dev traces for the same reason the databases are (see "Two stacks" in +[AGENTS.md](../AGENTS.md)): `docker compose --profile full down -v` deletes them along with your +financial data. -**Quick start (local):** +The production variables are `PROD_`-prefixed so a host-dev value in `.env` can never redirect +production traces: -```bash -# 1. Clone the Langfuse repository -git clone https://github.com/langfuse/langfuse.git -cd langfuse - -# 2. Update secrets in docker-compose.yml (lines marked with # CHANGEME) - -# 3. Start all services -docker compose up -``` - -After 2–3 minutes, `langfuse-web-1` will log "Ready". Open [http://localhost:3000](http://localhost:3000), create an account, and get your API keys from **Settings → API Keys**. +| Variable | Default | +| --------------------------------------------------------- | ---------------------------------- | +| `PROD_OTEL_EXPORTER_OTLP_ENDPOINT` | `http://phoenix-prod:6006` | +| `PROD_OTEL_EXPORTER_OTLP_HEADERS` | _(empty)_ | +| `PROD_OTEL_SDK_DISABLED` | `false` — `true` turns tracing off | +| `PROD_OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` | `false` | -> **Note:** Langfuse runs on port 3000 by default. Cameron's dev server uses **3100** (see -> `.env.example`), so there is no conflict — but if you've changed Cameron's port back to 3000, -> run one of them elsewhere (e.g. `pnpm dev --port 3200`). +Both Phoenix services bind to loopback only and run with Phoenix's usage telemetry disabled. +Phoenix has no login by default; don't publish its port. -Add to `.env.local` (adjust the port if needed): +## Configuration -```bash -LANGFUSE_ENABLED=true -LANGFUSE_SECRET_KEY="sk-lf-..." -LANGFUSE_PUBLIC_KEY="pk-lf-..." -LANGFUSE_BASE_URL="http://localhost:3000" -``` - -## Disabling Tracing +Everything is a standard OpenTelemetry variable: -Set `LANGFUSE_ENABLED=false` (or omit it entirely) in your `.env.local` and restart. No code changes required. There is zero performance overhead when disabled. +| Variable | Effect | +| ---------------------------------------------------- | ---------------------------------------------------------------------- | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Turns tracing on and sets the destination (`/v1/traces` is appended). | +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Same, but the full traces URL. | +| `OTEL_EXPORTER_OTLP_HEADERS` | Auth for backends that need it (`key=value,key2=value2`). | +| `OTEL_SERVICE_NAME` | Defaults to `cameron`. | +| `OTEL_SDK_DISABLED` | `true` turns tracing off even with an endpoint set. | +| `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` | `true` adds prompts, messages, tool arguments, results and error text. | -## What a Trace Looks Like +## What a trace contains -For a single user message that triggers a mutating tool call, you will see a trace in Langfuse with roughly this structure: +One user message (or one approval decision) is one trace: ``` -Trace: user-message - ├─ model node - │ └─ LLM call (model, input messages, output, token counts) - ├─ humanInTheLoopMiddleware - │ └─ interrupt (human-in-the-loop pause — mutating tools only) - ├─ tools node - │ └─ Tool execution (tool name, input args, output) - └─ model node (second pass) - └─ LLM call (final response generation) +GET /api/agent/stream Next.js route span +└─ executing api route (app) … + ├─ invoke_agent cameron the agent run (AGENT) + │ ├─ chat a model call (LLM) + │ ├─ execute_tool a tool call (TOOL) + │ └─ chat + └─ fetch POST Next.js's own fetch spans ``` -For a read-only tool the middleware interrupt is absent — the `tools` node runs straight after the first `model` node. - -## Key Files - -| File | Role | -| ------------------------------ | -------------------------------------------------------- | -| `instrumentation.ts` | Next.js OTel init hook — runs once before any route | -| `src/services/agentService.ts` | Attaches `CallbackHandler` to each `agent.stream()` call | -| `.env.example` | Documents all required environment variables | +Graph nodes and middleware get no span. **With content capture off**, these are the only +attributes Cameron sets: + +| Span | Attributes | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `invoke_agent` | `gen_ai.agent.name`, `gen_ai.conversation.id` (the thread id), `cameron.run.resumed`, and on a resume `cameron.approval.approved` / `.rejected` counts | +| `chat` | `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model`, `gen_ai.usage.input_tokens` / `output_tokens`, `gen_ai.response.finish_reasons` | +| `execute_tool` | `gen_ai.tool.name`, `gen_ai.tool.call.id`, `gen_ai.tool.type`, `cameron.tool.mutating` | +| any, on error | `error.type` and ERROR status — no message, no stack trace (an error message can quote an amount) | + +When the approval gate pauses a run, the `invoke_agent` span gets a `cameron.approval.requested` +event listing the gated **tool names** — never their arguments. A pause is not an error. + +**With content capture on**, Cameron adds `gen_ai.system_instructions`, `gen_ai.input.messages`, +`gen_ai.output.messages`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, the error +message and the exception event. On `invoke_agent`, the input is the turn's new user message (a +resume has none — its input is a decision) and the output is the run's last assistant message. Image and file blocks are recorded by type only, never inlined. +Turn this on to debug, not by default: the result of a `run_sql` or `query_transactions` call is +your ledger. + +The thread id is metadata, not content, so it is always sent; it is a random id, not a name. + +## Other backends + +The spans are plain OTLP, so a different backend is an endpoint change: + +| Backend | Run it | Endpoint | +| ------------------------ | ---------------------------------------------------------------------------------------- | --------------------------------------- | +| Jaeger (generic tracing) | `docker run -p 127.0.0.1:16686:16686 -p 127.0.0.1:4318:4318 jaegertracing/jaeger:2.11.0` | `http://localhost:4318` | +| Self-hosted Langfuse | [docker compose guide](https://langfuse.com/self-hosting/deployment/docker-compose) | `http://localhost:3000/api/public/otel` | + +Langfuse also needs `OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic%20` +(header values are URL-encoded, so the space is `%20`). + +Jaeger shows the same tree but no LLM-specific view (messages, token counts appear as raw +attributes). Phoenix and Langfuse understand the GenAI conventions natively. A hosted backend also +works, but then your traces are no longer on infrastructure you control — keep content capture off +if you do. + +## How it's built + +| File | Role | +| -------------------------------- | ------------------------------------------------------------------------------------- | +| `instrumentation.ts` | Starts the OTel Node SDK with the OTLP/proto exporter, only when tracing is enabled | +| `src/lib/tracing/config.ts` | `tracingEnabled()` and `captureContent()`, read from the environment | +| `src/lib/tracing/genAiTracer.ts` | LangChain callback handler that turns one run's callbacks into spans; one per request | +| `src/lib/tracing/semconv.ts` | The only file naming `gen_ai.*` and `cameron.*` attributes | +| `src/lib/tracing/messages.ts` | LangChain messages → the semconv message format (content capture only) | +| `src/services/agentService.ts` | Attaches a `GenAiTracer` to each run when tracing is enabled | + +Notes for changing it: + +- **The GenAI conventions are still at Development stability.** Every attribute name lives in + `semconv.ts`; a rename is a one-file change. Phoenix (20.23) converts these attributes to its own + OpenInference format at ingest, which is how it shows span kinds, sessions and token counts. +- **LangChain's callback arguments don't match its type definitions.** `handleChainStart` is called + with `(chain, inputs, runId, parentRunId, tags, metadata, …)`; the `.d.ts` puts `parentRunId` + eighth. `genAiTracer.ts` follows the runtime order. +- **The approval pause is only visible on the HITL node.** It fails with `GraphInterrupt` (which + carries the gated tool names), while the run itself ends normally. A resumed run's input is the + `Command`, which is where the decision counts come from. +- **Model `fetch` spans are not children of the `chat` span.** Next.js creates them in the route's + context; a callback handler can open spans but can't make them the active context for code it + doesn't call. +- Unit tests (`genAiTracer.test.ts`) drive the handler with hand-fed callbacks against an + in-memory exporter — free and offline like the rest of `pnpm test`. diff --git a/instrumentation.ts b/instrumentation.ts index 2f4443c..d09e424 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,17 +1,18 @@ export async function register() { - // Only initialize in the Node.js runtime — @opentelemetry/sdk-node uses - // Node.js-only APIs that are not available in the Edge runtime. - // Next.js may evaluate this file in both contexts. - if (process.env.NEXT_RUNTIME === "nodejs" && process.env.LANGFUSE_ENABLED === "true") { - // Dynamic imports are required here so Next.js doesn't attempt to bundle - // Node.js-only modules for the Edge runtime. - const { NodeSDK } = await import("@opentelemetry/sdk-node"); - const { LangfuseSpanProcessor } = await import("@langfuse/otel"); + // @opentelemetry/sdk-node is Node-only; Next.js also evaluates this file for the Edge runtime. + if (process.env.NEXT_RUNTIME !== "nodejs") return; - const sdk = new NodeSDK({ - spanProcessors: [new LangfuseSpanProcessor()], - }); + const { tracingEnabled } = await import("./src/lib/tracing/config"); + if (!tracingEnabled()) return; - sdk.start(); - } + const { NodeSDK } = await import("@opentelemetry/sdk-node"); + const { OTLPTraceExporter } = await import("@opentelemetry/exporter-trace-otlp-proto"); + + // Endpoint and headers come from the standard OTEL_EXPORTER_OTLP_* variables. + const sdk = new NodeSDK({ + serviceName: process.env.OTEL_SERVICE_NAME ?? "cameron", + traceExporter: new OTLPTraceExporter(), + }); + sdk.start(); + process.once("SIGTERM", () => void sdk.shutdown()); } diff --git a/package.json b/package.json index 76d896d..045c5a3 100644 --- a/package.json +++ b/package.json @@ -35,9 +35,9 @@ "@langchain/langgraph-checkpoint-postgres": "^1.0.3", "@langchain/mcp-adapters": "^1.1.3", "@langchain/openai": "^1.5.2", - "@langfuse/langchain": "^5.5.3", - "@langfuse/otel": "^5.5.3", "@modelcontextprotocol/sdk": "^1.29.0", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/exporter-trace-otlp-proto": "^0.219.0", "@opentelemetry/sdk-node": "^0.219.0", "@radix-ui/react-scroll-area": "^1.2.12", "@radix-ui/react-separator": "^1.1.10", @@ -67,6 +67,7 @@ }, "devDependencies": { "@eslint/eslintrc": "^3.3.5", + "@opentelemetry/sdk-trace-base": "^2.8.0", "@shadcn/ui": "^0.0.4", "@tailwindcss/postcss": "^4.3.1", "@tailwindcss/typography": "^0.5.20", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7463e0c..831e676 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -52,15 +52,15 @@ importers: '@langchain/openai': specifier: ^1.5.2 version: 1.5.2(@langchain/core@1.2.1(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.219.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1))(openai@6.44.0(ws@8.18.3)(zod@4.4.3))(ws@8.18.3))(ws@8.18.3) - '@langfuse/langchain': - specifier: ^5.5.3 - version: 5.5.3(@langchain/core@1.2.1(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.219.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1))(openai@6.44.0(ws@8.18.3)(zod@4.4.3))(ws@8.18.3))(@opentelemetry/api@1.9.1) - '@langfuse/otel': - specifier: ^5.5.3 - version: 5.5.3(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.219.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1)) '@modelcontextprotocol/sdk': specifier: ^1.29.0 version: 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3) + '@opentelemetry/api': + specifier: ^1.9.1 + version: 1.9.1 + '@opentelemetry/exporter-trace-otlp-proto': + specifier: ^0.219.0 + version: 0.219.0(@opentelemetry/api@1.9.1) '@opentelemetry/sdk-node': specifier: ^0.219.0 version: 0.219.0(@opentelemetry/api@1.9.1) @@ -143,6 +143,9 @@ importers: '@eslint/eslintrc': specifier: ^3.3.5 version: 3.3.5 + '@opentelemetry/sdk-trace-base': + specifier: ^2.8.0 + version: 2.8.0(@opentelemetry/api@1.9.1) '@shadcn/ui': specifier: ^0.0.4 version: 0.0.4 @@ -1241,32 +1244,6 @@ packages: '@langchain/protocol@0.0.18': resolution: {integrity: sha512-XW1egQtPfsGI41w2AMZNFZrUIwFSQHTjVMZs0OaTpCAvht/QLoaPN8FQcsysMVypOhupG28J29yOorrc70otBQ==} - '@langfuse/core@5.5.3': - resolution: {integrity: sha512-F2auGm6l4/QhQ6Y86+f6QqulXNetVsaI3u7ccyqADEtGcDWk1+faSczxuSMJ0X/EfrjVZ+98Luy/FEEah+HDRA==} - peerDependencies: - '@opentelemetry/api': ^1.9.0 - - '@langfuse/langchain@5.5.3': - resolution: {integrity: sha512-uxTuLuVV+KAldlk05lQUzD8ADFSvNdRkc5sxa1T4Sguttkpk3Sxd9hC0/QlsEr/s9hb2IIqjeSnojRQjOsp/vA==} - peerDependencies: - '@langchain/core': '>=0.3.8' - '@opentelemetry/api': ^1.9.0 - - '@langfuse/otel@5.5.3': - resolution: {integrity: sha512-j0nPAEFHKorEV8JtBRzZaSShUbjLYk7omtxXxy1bLvNb5QFGO1gF7HFm+fSqnoN70034EvTXYCNFDWpBrAH/vw==} - engines: {node: '>=20'} - peerDependencies: - '@opentelemetry/api': ^1.9.0 - '@opentelemetry/core': ^2.0.1 - '@opentelemetry/exporter-trace-otlp-http': '>=0.202.0 <1.0.0' - '@opentelemetry/sdk-trace-base': ^2.0.1 - - '@langfuse/tracing@5.5.3': - resolution: {integrity: sha512-t/WoFCOtXjdFSFx7A4+nesX0eB4c0gP7ca7j6kv38kIubPE+IniHOhNYPXdAdm5tc3EL+P+gpMCtibCBp8xYsQ==} - engines: {node: '>=20'} - peerDependencies: - '@opentelemetry/api': ^1.9.0 - '@modelcontextprotocol/sdk@1.29.0': resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} engines: {node: '>=18'} @@ -6292,30 +6269,6 @@ snapshots: '@langchain/protocol@0.0.18': {} - '@langfuse/core@5.5.3(@opentelemetry/api@1.9.1)': - dependencies: - '@opentelemetry/api': 1.9.1 - - '@langfuse/langchain@5.5.3(@langchain/core@1.2.1(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.219.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1))(openai@6.44.0(ws@8.18.3)(zod@4.4.3))(ws@8.18.3))(@opentelemetry/api@1.9.1)': - dependencies: - '@langchain/core': 1.2.1(@opentelemetry/api@1.9.1)(@opentelemetry/exporter-trace-otlp-proto@0.219.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1))(openai@6.44.0(ws@8.18.3)(zod@4.4.3))(ws@8.18.3) - '@langfuse/core': 5.5.3(@opentelemetry/api@1.9.1) - '@langfuse/tracing': 5.5.3(@opentelemetry/api@1.9.1) - '@opentelemetry/api': 1.9.1 - - '@langfuse/otel@5.5.3(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.8.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.219.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.8.0(@opentelemetry/api@1.9.1))': - dependencies: - '@langfuse/core': 5.5.3(@opentelemetry/api@1.9.1) - '@opentelemetry/api': 1.9.1 - '@opentelemetry/core': 2.8.0(@opentelemetry/api@1.9.1) - '@opentelemetry/exporter-trace-otlp-http': 0.219.0(@opentelemetry/api@1.9.1) - '@opentelemetry/sdk-trace-base': 2.8.0(@opentelemetry/api@1.9.1) - - '@langfuse/tracing@5.5.3(@opentelemetry/api@1.9.1)': - dependencies: - '@langfuse/core': 5.5.3(@opentelemetry/api@1.9.1) - '@opentelemetry/api': 1.9.1 - '@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.4.3)': dependencies: '@hono/node-server': 1.19.12(hono@4.12.11) diff --git a/src/lib/tracing/config.test.ts b/src/lib/tracing/config.test.ts new file mode 100644 index 0000000..6693920 --- /dev/null +++ b/src/lib/tracing/config.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { captureContent, tracingEnabled } from "./config"; + +describe("tracingEnabled", () => { + it("is off when no OTLP endpoint is set", () => { + expect(tracingEnabled({})).toBe(false); + }); + + it("is on for either the generic or the traces endpoint", () => { + expect(tracingEnabled({ OTEL_EXPORTER_OTLP_ENDPOINT: "http://phoenix:6006" })).toBe(true); + expect( + tracingEnabled({ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://phoenix:6006/v1/traces" }), + ).toBe(true); + }); + + it("respects OTEL_SDK_DISABLED even with an endpoint", () => { + expect( + tracingEnabled({ + OTEL_EXPORTER_OTLP_ENDPOINT: "http://phoenix:6006", + OTEL_SDK_DISABLED: "true", + }), + ).toBe(false); + }); +}); + +describe("captureContent", () => { + it("is off unless explicitly enabled", () => { + expect(captureContent({})).toBe(false); + expect(captureContent({ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "1" })).toBe(false); + expect(captureContent({ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "true" })).toBe( + true, + ); + }); +}); diff --git a/src/lib/tracing/config.ts b/src/lib/tracing/config.ts new file mode 100644 index 0000000..b74dc02 --- /dev/null +++ b/src/lib/tracing/config.ts @@ -0,0 +1,12 @@ +type Env = Record; + +/** Tracing is on only when the owner names an OTLP endpoint; there is no default destination. */ +export function tracingEnabled(env: Env = process.env): boolean { + if (env.OTEL_SDK_DISABLED === "true") return false; + return Boolean(env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || env.OTEL_EXPORTER_OTLP_ENDPOINT); +} + +/** Prompts, messages, tool arguments and results leave the process only when this is "true". */ +export function captureContent(env: Env = process.env): boolean { + return env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT === "true"; +} diff --git a/src/lib/tracing/genAiTracer.test.ts b/src/lib/tracing/genAiTracer.test.ts new file mode 100644 index 0000000..150197a --- /dev/null +++ b/src/lib/tracing/genAiTracer.test.ts @@ -0,0 +1,405 @@ +import { describe, expect, it } from "vitest"; +import { AIMessage, HumanMessage, SystemMessage, ToolMessage } from "@langchain/core/messages"; +import type { Serialized } from "@langchain/core/load/serializable"; +import { Command, GraphInterrupt } from "@langchain/langgraph"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, + type ReadableSpan, +} from "@opentelemetry/sdk-trace-base"; +import { GenAiTracer } from "./genAiTracer"; + +const SERIALIZED = {} as Serialized; +const CONTENT_KEYS = [ + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.system_instructions", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", +]; + +function setup(captureContent = false) { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const tracer = new GenAiTracer({ tracer: provider.getTracer("test"), captureContent }); + const spans = () => exporter.getFinishedSpans(); + const byName = (name: string) => spans().find((s) => s.name === name) as ReadableSpan; + return { tracer, spans, byName }; +} + +const META = { ls_provider: "anthropic", ls_model_name: "claude-haiku-5-5", thread_id: "thread-1" }; + +function aiReply(fields: ConstructorParameters[0]) { + return { generations: [[{ text: "", message: new AIMessage(fields) }]] }; +} + +function startRoot(t: GenAiTracer, inputs: Record = { messages: [] }) { + t.handleChainStart(SERIALIZED, inputs, "root", undefined, [], { thread_id: "thread-1" }); +} + +function chat(t: GenAiTracer, runId: string, parent: string, reply = aiReply({ content: "ok" })) { + t.handleChatModelStart( + SERIALIZED, + [[new SystemMessage("You are Cameron."), new HumanMessage("What categories do I have?")]], + runId, + parent, + { invocation_params: { model: "claude-haiku-5-5" } }, + [], + META, + ); + t.handleLLMEnd(reply, runId); +} + +function tool(t: GenAiTracer, runId: string, parent: string, name: string, output: unknown) { + t.handleToolStart(SERIALIZED, '{"amount":12.5}', runId, parent, [], {}, name, `call-${runId}`); + t.handleToolEnd(output, runId); +} + +const QUESTION = new HumanMessage("What categories do I have?"); +const ANSWER = new AIMessage({ + content: "You have one category: Dining.", + response_metadata: { finish_reason: "stop" }, +}); + +function readRun(t: GenAiTracer) { + startRoot(t, { messages: [QUESTION] }); + t.handleChainStart(SERIALIZED, {}, "node-model", "root"); + chat( + t, + "chat-1", + "node-model", + aiReply({ + content: "", + tool_calls: [{ id: "call-tool-1", name: "list_categories", args: {} }], + usage_metadata: { input_tokens: 12913, output_tokens: 163, total_tokens: 13076 }, + response_metadata: { finish_reason: "tool_use", model_name: "claude-haiku-5-5-20261001" }, + }), + ); + t.handleChainEnd({}, "node-model"); + t.handleChainStart(SERIALIZED, {}, "node-tools", "root"); + tool( + t, + "tool-1", + "node-tools", + "list_categories", + new ToolMessage({ content: '{"categories":["Dining"]}', tool_call_id: "call-tool-1" }), + ); + t.handleChainEnd({}, "node-tools"); + t.handleChainEnd( + { + messages: [ + new HumanMessage("Earlier question"), + new AIMessage("Earlier answer"), + QUESTION, + ANSWER, + ], + }, + "root", + ); +} + +describe("GenAiTracer span tree", () => { + it("emits one agent span with the model and tool spans as its direct children", () => { + const { tracer, spans, byName } = setup(); + readRun(tracer); + + expect( + spans() + .map((s) => s.name) + .sort(), + ).toEqual(["chat claude-haiku-5-5", "execute_tool list_categories", "invoke_agent cameron"]); + const root = byName("invoke_agent cameron"); + for (const name of ["chat claude-haiku-5-5", "execute_tool list_categories"]) { + expect(byName(name).parentSpanContext?.spanId).toBe(root.spanContext().spanId); + expect(byName(name).spanContext().traceId).toBe(root.spanContext().traceId); + } + }); + + it("labels the agent span with the conversation and marks a fresh run as not resumed", () => { + const { tracer, byName } = setup(); + readRun(tracer); + + expect(byName("invoke_agent cameron").attributes).toMatchObject({ + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "cameron", + "gen_ai.conversation.id": "thread-1", + "cameron.run.resumed": false, + }); + }); + + it("records provider, models, token usage and finish reason on the model span", () => { + const { tracer, byName } = setup(); + readRun(tracer); + + expect(byName("chat claude-haiku-5-5").attributes).toMatchObject({ + "gen_ai.operation.name": "chat", + "gen_ai.provider.name": "anthropic", + "gen_ai.request.model": "claude-haiku-5-5", + "gen_ai.response.model": "claude-haiku-5-5-20261001", + "gen_ai.usage.input_tokens": 12913, + "gen_ai.usage.output_tokens": 163, + "gen_ai.response.finish_reasons": ["tool_use"], + }); + }); + + it("omits token usage when the provider reports none", () => { + const { tracer, byName } = setup(); + startRoot(tracer); + chat(tracer, "chat-1", "root"); + tracer.handleChainEnd({}, "root"); + + const attrs = byName("chat claude-haiku-5-5").attributes; + expect(attrs).not.toHaveProperty("gen_ai.usage.input_tokens"); + expect(attrs).not.toHaveProperty("gen_ai.usage.output_tokens"); + }); + + it("maps LangChain's Google provider id to the semconv name", () => { + const { tracer, spans } = setup(); + startRoot(tracer); + tracer.handleChatModelStart(SERIALIZED, [[]], "chat-1", "root", {}, [], { + ls_provider: "google_genai", + ls_model_name: "gemini", + }); + tracer.handleLLMEnd(aiReply({ content: "" }), "chat-1"); + tracer.handleChainEnd({}, "root"); + + expect(spans().find((s) => s.name === "chat gemini")?.attributes["gen_ai.provider.name"]).toBe( + "gcp.gemini", + ); + }); + + it("flags mutating tools so they can be told apart from reads", () => { + const { tracer, byName } = setup(); + startRoot(tracer); + tool( + tracer, + "t1", + "root", + "log_expense", + new ToolMessage({ content: "{}", tool_call_id: "c1" }), + ); + tool( + tracer, + "t2", + "root", + "list_categories", + new ToolMessage({ content: "{}", tool_call_id: "c2" }), + ); + tracer.handleChainEnd({}, "root"); + + expect(byName("execute_tool log_expense").attributes).toMatchObject({ + "cameron.tool.mutating": true, + "gen_ai.tool.call.id": "call-t1", + "gen_ai.tool.type": "function", + }); + expect(byName("execute_tool list_categories").attributes["cameron.tool.mutating"]).toBe(false); + }); +}); + +describe("GenAiTracer content capture", () => { + it("sends no prompts, messages, arguments or results when capture is off", () => { + const { tracer, spans } = setup(false); + readRun(tracer); + + for (const span of spans()) { + for (const key of CONTENT_KEYS) expect(span.attributes).not.toHaveProperty(key); + } + }); + + it("sends semconv messages, system instructions, arguments and results when capture is on", () => { + const { tracer, byName } = setup(true); + readRun(tracer); + + const chatAttrs = byName("chat claude-haiku-5-5").attributes; + expect(JSON.parse(chatAttrs["gen_ai.system_instructions"] as string)).toEqual([ + { type: "text", content: "You are Cameron." }, + ]); + expect(JSON.parse(chatAttrs["gen_ai.input.messages"] as string)).toEqual([ + { role: "user", parts: [{ type: "text", content: "What categories do I have?" }] }, + ]); + expect(JSON.parse(chatAttrs["gen_ai.output.messages"] as string)).toEqual([ + { + role: "assistant", + parts: [{ type: "tool_call", id: "call-tool-1", name: "list_categories", arguments: {} }], + finish_reason: "tool_use", + }, + ]); + expect(byName("execute_tool list_categories").attributes).toMatchObject({ + "gen_ai.tool.call.arguments": '{"amount":12.5}', + "gen_ai.tool.call.result": '{"categories":["Dining"]}', + }); + }); + + it("records the run's new messages and final answer on the agent span", () => { + const { tracer, byName } = setup(true); + readRun(tracer); + + const attrs = byName("invoke_agent cameron").attributes; + expect(JSON.parse(attrs["gen_ai.input.messages"] as string)).toEqual([ + { role: "user", parts: [{ type: "text", content: "What categories do I have?" }] }, + ]); + expect(JSON.parse(attrs["gen_ai.output.messages"] as string)).toEqual([ + { + role: "assistant", + parts: [{ type: "text", content: "You have one category: Dining." }], + finish_reason: "stop", + }, + ]); + }); + + it("records no input on a resumed run, whose input is a decision rather than a message", () => { + const { tracer, byName } = setup(true); + startRoot(tracer, new Command({ resume: { decisions: [{ type: "approve" }] } }) as never); + tracer.handleChainEnd({ messages: [ANSWER] }, "root"); + + const attrs = byName("invoke_agent cameron").attributes; + expect(attrs).not.toHaveProperty("gen_ai.input.messages"); + expect(attrs).toHaveProperty("gen_ai.output.messages"); + }); + + it("names non-text content blocks without inlining their data", () => { + const { tracer, byName } = setup(true); + startRoot(tracer); + tracer.handleChatModelStart( + SERIALIZED, + [ + [ + new HumanMessage({ + content: [ + { type: "text", text: "Read this receipt" }, + { type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } }, + ], + }), + ], + ], + "chat-1", + "root", + {}, + [], + META, + ); + tracer.handleLLMEnd(aiReply({ content: "" }), "chat-1"); + tracer.handleChainEnd({}, "root"); + + const input = byName("chat claude-haiku-5-5").attributes["gen_ai.input.messages"] as string; + expect(JSON.parse(input)[0].parts).toEqual([ + { type: "text", content: "Read this receipt" }, + { type: "image_url" }, + ]); + expect(input).not.toContain("base64"); + }); +}); + +describe("GenAiTracer approval gate", () => { + function pause(t: GenAiTracer) { + startRoot(t); + t.handleChainStart(SERIALIZED, {}, "hitl", "root"); + t.handleChainError( + new GraphInterrupt([ + { + id: "i1", + value: { + actionRequests: [{ name: "log_expense", args: { amount: 12.5, note: "coffee" } }], + }, + }, + ]), + "hitl", + ); + t.handleChainEnd({ messages: [] }, "root"); + } + + it("records the approval request on the agent span without arguments or an error", () => { + const { tracer, byName } = setup(true); + pause(tracer); + + const root = byName("invoke_agent cameron"); + expect(root.status.code).not.toBe(SpanStatusCode.ERROR); + expect(root.events.map((e) => e.name)).toEqual(["cameron.approval.requested"]); + expect(root.events[0].attributes).toEqual({ "cameron.approval.tools": ["log_expense"] }); + expect(JSON.stringify(root.events)).not.toContain("coffee"); + }); + + it("marks a resumed run with its approve and reject counts", () => { + const { tracer, byName } = setup(); + startRoot( + tracer, + new Command({ + resume: { decisions: [{ type: "approve" }, { type: "reject" }, { type: "reject" }] }, + }) as never, + ); + tracer.handleChainEnd({}, "root"); + + expect(byName("invoke_agent cameron").attributes).toMatchObject({ + "cameron.run.resumed": true, + "cameron.approval.approved": 1, + "cameron.approval.rejected": 2, + }); + }); +}); + +describe("GenAiTracer errors", () => { + function failingTool(t: GenAiTracer) { + startRoot(t); + t.handleToolStart(SERIALIZED, "{}", "tool-1", "root", [], {}, "run_sql", "c1"); + t.handleToolError(new Error("amount 1234.56 is invalid"), "tool-1"); + t.handleChainEnd({}, "root"); + } + + it("marks a thrown tool error without its message when capture is off", () => { + const { tracer, byName } = setup(false); + failingTool(tracer); + + const span = byName("execute_tool run_sql"); + expect(span.status).toEqual({ code: SpanStatusCode.ERROR }); + expect(span.attributes["error.type"]).toBe("Error"); + expect(span.events).toEqual([]); + }); + + it("keeps the error message and exception when capture is on", () => { + const { tracer, byName } = setup(true); + failingTool(tracer); + + const span = byName("execute_tool run_sql"); + expect(span.status).toEqual({ + code: SpanStatusCode.ERROR, + message: "amount 1234.56 is invalid", + }); + expect(span.events.map((e) => e.name)).toEqual(["exception"]); + }); + + it("marks a tool that returned an error message as failed", () => { + const { tracer, byName } = setup(); + startRoot(tracer); + tool( + tracer, + "t1", + "root", + "log_expense", + new ToolMessage({ content: "boom", tool_call_id: "c1", status: "error" }), + ); + tracer.handleChainEnd({}, "root"); + + expect(byName("execute_tool log_expense").status.code).toBe(SpanStatusCode.ERROR); + expect(byName("execute_tool log_expense").attributes["error.type"]).toBe("tool_error"); + }); + + it("marks the agent span failed when the run itself throws", () => { + const { tracer, byName } = setup(); + startRoot(tracer); + tracer.handleChainError(new Error("recursion limit"), "root"); + + expect(byName("invoke_agent cameron").status.code).toBe(SpanStatusCode.ERROR); + }); + + it("closes spans left open when the run ends and flags them unfinished", () => { + const { tracer, spans, byName } = setup(); + startRoot(tracer); + tracer.handleToolStart(SERIALIZED, "{}", "tool-1", "root", [], {}, "run_sql", "c1"); + tracer.handleChainError(new Error("aborted"), "root"); + + expect(spans()).toHaveLength(2); + expect(byName("execute_tool run_sql").attributes["cameron.span.unfinished"]).toBe(true); + }); +}); diff --git a/src/lib/tracing/genAiTracer.ts b/src/lib/tracing/genAiTracer.ts new file mode 100644 index 0000000..f2eb786 --- /dev/null +++ b/src/lib/tracing/genAiTracer.ts @@ -0,0 +1,303 @@ +import { BaseCallbackHandler } from "@langchain/core/callbacks/base"; +import type { Serialized } from "@langchain/core/load/serializable"; +import { AIMessage, isBaseMessage, ToolMessage, type BaseMessage } from "@langchain/core/messages"; +import type { ChatGeneration, LLMResult } from "@langchain/core/outputs"; +import type { ChainValues } from "@langchain/core/utils/types"; +import { isGraphInterrupt } from "@langchain/langgraph"; +import { + context, + SpanKind, + SpanStatusCode, + trace, + type Attributes, + type Context, + type Span, + type Tracer, +} from "@opentelemetry/api"; +import { isMutatingTool } from "@/lib/agent/mutatingTools"; +import { captureContent } from "./config"; +import { toSemconvInput, toSemconvOutput } from "./messages"; +import { AGENT_NAME, CAMERON, ERROR_TYPE, GEN_AI, OPERATION, toProviderName } from "./semconv"; + +type Metadata = Record; + +const str = (v: unknown): string | undefined => (typeof v === "string" && v ? v : undefined); + +// One agent run's callbacks → OTel GenAI spans (run, model calls, tool calls). One per request. +export class GenAiTracer extends BaseCallbackHandler { + name = "cameron_genai_tracer"; + + private readonly tracer: Tracer; + private readonly capture: boolean; + private readonly spans = new Map(); + private readonly parents = new Map(); + private rootRunId: string | undefined; + private approvalRecorded = false; + + constructor(opts: { tracer?: Tracer; captureContent?: boolean } = {}) { + super(); + this.tracer = opts.tracer ?? trace.getTracer("cameron"); + this.capture = opts.captureContent ?? captureContent(); + } + + // LangChain calls this with (…, runId, parentRunId, tags, metadata), not the order its .d.ts declares. + handleChainStart( + _chain: Serialized, + inputs: ChainValues, + runId: string, + parentRunId?: string, + _tags?: string[], + metadata?: Metadata, + ) { + this.parents.set(runId, parentRunId); + if (parentRunId) return; + + this.rootRunId = runId; + this.approvalRecorded = false; + const conversationId = str(metadata?.thread_id); + const span = this.tracer.startSpan( + `${OPERATION.INVOKE_AGENT} ${AGENT_NAME}`, + { + kind: SpanKind.INTERNAL, + attributes: { + [GEN_AI.OPERATION_NAME]: OPERATION.INVOKE_AGENT, + [GEN_AI.AGENT_NAME]: AGENT_NAME, + ...(conversationId ? { [GEN_AI.CONVERSATION_ID]: conversationId } : {}), + ...resumeAttributes(inputs), + ...(this.capture ? runInputAttributes(inputs) : {}), + }, + }, + context.active(), + ); + this.spans.set(runId, span); + } + + handleChainEnd(outputs: ChainValues, runId: string) { + if (runId !== this.rootRunId) return; + const root = this.spans.get(runId); + if (root && this.capture) root.setAttributes(runOutputAttributes(outputs)); + this.finishRoot(); + } + + handleChainError(err: Error, runId: string) { + const interrupted = isGraphInterrupt(err); + if (interrupted) this.recordApprovalRequest(err); + if (runId !== this.rootRunId) return; + const root = this.spans.get(runId); + if (root && !interrupted) this.fail(root, err); + this.finishRoot(); + } + + handleChatModelStart( + _llm: Serialized, + messages: BaseMessage[][], + runId: string, + parentRunId?: string, + extraParams?: Metadata, + _tags?: string[], + metadata?: Metadata, + ) { + this.parents.set(runId, parentRunId); + const invocation = (extraParams?.invocation_params ?? {}) as Metadata; + const model = str(metadata?.ls_model_name) ?? str(invocation.model); + const provider = str(metadata?.ls_provider); + + const attributes: Attributes = { + [GEN_AI.OPERATION_NAME]: OPERATION.CHAT, + ...(provider ? { [GEN_AI.PROVIDER_NAME]: toProviderName(provider) } : {}), + ...(model ? { [GEN_AI.REQUEST_MODEL]: model } : {}), + }; + if (this.capture) { + const input = toSemconvInput(messages[0] ?? []); + if (input.systemInstructions.length) { + attributes[GEN_AI.SYSTEM_INSTRUCTIONS] = JSON.stringify(input.systemInstructions); + } + attributes[GEN_AI.INPUT_MESSAGES] = JSON.stringify(input.messages); + } + + const name = model ? `${OPERATION.CHAT} ${model}` : OPERATION.CHAT; + this.start(runId, parentRunId, name, SpanKind.CLIENT, attributes); + } + + handleLLMEnd(output: LLMResult, runId: string) { + const span = this.spans.get(runId); + if (!span) return; + const generation = output.generations?.[0]?.[0] as ChatGeneration | undefined; + const message = generation?.message; + const usage = (message as { usage_metadata?: Metadata } | undefined)?.usage_metadata; + const meta = (message?.response_metadata ?? {}) as Metadata; + const finishReason = finishReasonOf(message) ?? str(generation?.generationInfo?.finish_reason); + const responseModel = str(meta.model_name) ?? str(meta.model); + + if (typeof usage?.input_tokens === "number") { + span.setAttribute(GEN_AI.USAGE_INPUT_TOKENS, usage.input_tokens); + } + if (typeof usage?.output_tokens === "number") { + span.setAttribute(GEN_AI.USAGE_OUTPUT_TOKENS, usage.output_tokens); + } + if (responseModel) span.setAttribute(GEN_AI.RESPONSE_MODEL, responseModel); + if (finishReason) span.setAttribute(GEN_AI.RESPONSE_FINISH_REASONS, [finishReason]); + if (this.capture && message) { + span.setAttribute( + GEN_AI.OUTPUT_MESSAGES, + JSON.stringify(toSemconvOutput(message, finishReason)), + ); + } + this.end(runId); + } + + handleLLMError(err: Error, runId: string) { + const span = this.spans.get(runId); + if (span) this.fail(span, err); + this.end(runId); + } + + handleToolStart( + _tool: Serialized, + input: string, + runId: string, + parentRunId?: string, + _tags?: string[], + _metadata?: Metadata, + runName?: string, + toolCallId?: string, + ) { + this.parents.set(runId, parentRunId); + const toolName = runName ?? "unknown"; + const attributes: Attributes = { + [GEN_AI.OPERATION_NAME]: OPERATION.EXECUTE_TOOL, + [GEN_AI.TOOL_NAME]: toolName, + [GEN_AI.TOOL_TYPE]: "function", + [CAMERON.TOOL_MUTATING]: isMutatingTool(toolName), + ...(toolCallId ? { [GEN_AI.TOOL_CALL_ID]: toolCallId } : {}), + ...(this.capture ? { [GEN_AI.TOOL_CALL_ARGUMENTS]: input } : {}), + }; + this.start( + runId, + parentRunId, + `${OPERATION.EXECUTE_TOOL} ${toolName}`, + SpanKind.INTERNAL, + attributes, + ); + } + + handleToolEnd(output: unknown, runId: string) { + const span = this.spans.get(runId); + if (!span) return; + const message = ToolMessage.isInstance(output) ? output : undefined; + if (message?.status === "error") { + span.setAttribute(ERROR_TYPE, "tool_error"); + span.setStatus({ code: SpanStatusCode.ERROR }); + } + if (this.capture) + span.setAttribute(GEN_AI.TOOL_CALL_RESULT, toolResult(message?.content ?? output)); + this.end(runId); + } + + handleToolError(err: Error, runId: string) { + const span = this.spans.get(runId); + if (isGraphInterrupt(err)) this.recordApprovalRequest(err); + else if (span) this.fail(span, err); + this.end(runId); + } + + private start( + runId: string, + parentRunId: string | undefined, + name: string, + kind: SpanKind, + attributes: Attributes, + ) { + const span = this.tracer.startSpan(name, { kind, attributes }, this.parentContext(parentRunId)); + this.spans.set(runId, span); + } + + private end(runId: string) { + this.spans.get(runId)?.end(); + this.spans.delete(runId); + } + + // Graph nodes and middleware get no span, so walk up to the nearest run that has one. + private parentContext(parentRunId: string | undefined): Context { + for (let id = parentRunId; id; id = this.parents.get(id)) { + const span = this.spans.get(id); + if (span) return trace.setSpan(context.active(), span); + } + return context.active(); + } + + private fail(span: Span, err: Error) { + span.setAttribute(ERROR_TYPE, err?.name || "Error"); + // An error message can quote a tool's arguments, so it is content too. + span.setStatus({ + code: SpanStatusCode.ERROR, + ...(this.capture ? { message: err?.message } : {}), + }); + if (this.capture) span.recordException(err); + } + + private recordApprovalRequest(err: unknown) { + const root = this.rootRunId ? this.spans.get(this.rootRunId) : undefined; + if (!root || this.approvalRecorded) return; + const tools = ((err as { interrupts?: { value?: unknown }[] }).interrupts ?? []).flatMap( + (i) => { + const requests = (i.value as { actionRequests?: { name?: unknown }[] } | undefined) + ?.actionRequests; + return (requests ?? []).map((r) => String(r.name)); + }, + ); + if (tools.length === 0) return; + this.approvalRecorded = true; + root.addEvent(CAMERON.APPROVAL_REQUESTED_EVENT, { [CAMERON.APPROVAL_TOOLS]: tools }); + } + + private finishRoot() { + const rootId = this.rootRunId; + for (const [id, span] of this.spans) { + if (id === rootId) continue; + span.setAttribute(CAMERON.SPAN_UNFINISHED, true); + span.end(); + } + if (rootId) this.spans.get(rootId)?.end(); + this.spans.clear(); + this.parents.clear(); + this.rootRunId = undefined; + } +} + +function resumeAttributes(inputs: ChainValues): Attributes { + const decisions = (inputs as { resume?: { decisions?: { type?: string }[] } } | undefined)?.resume + ?.decisions; + if (!Array.isArray(decisions)) return { [CAMERON.RUN_RESUMED]: false }; + return { + [CAMERON.RUN_RESUMED]: true, + [CAMERON.APPROVALS_APPROVED]: decisions.filter((d) => d?.type === "approve").length, + [CAMERON.APPROVALS_REJECTED]: decisions.filter((d) => d?.type === "reject").length, + }; +} + +const messagesOf = (values: ChainValues | undefined): BaseMessage[] => + Array.isArray(values?.messages) ? values.messages.filter(isBaseMessage) : []; + +// The turn's new messages; a resume carries a Command instead, so it has none. +function runInputAttributes(inputs: ChainValues): Attributes { + const messages = messagesOf(inputs); + if (messages.length === 0) return {}; + return { [GEN_AI.INPUT_MESSAGES]: JSON.stringify(toSemconvInput(messages).messages) }; +} + +// The output state holds the whole thread; the run's answer is its last AI message. +function runOutputAttributes(outputs: ChainValues): Attributes { + const last = messagesOf(outputs).findLast((m) => AIMessage.isInstance(m)); + if (!last) return {}; + return { [GEN_AI.OUTPUT_MESSAGES]: JSON.stringify(toSemconvOutput(last, finishReasonOf(last))) }; +} + +function finishReasonOf(message: BaseMessage | undefined): string | undefined { + const meta = (message?.response_metadata ?? {}) as Metadata; + return str(meta.finish_reason) ?? str(meta.stop_reason); +} + +function toolResult(content: unknown): string { + return typeof content === "string" ? content : JSON.stringify(content); +} diff --git a/src/lib/tracing/messages.ts b/src/lib/tracing/messages.ts new file mode 100644 index 0000000..2d2f1a5 --- /dev/null +++ b/src/lib/tracing/messages.ts @@ -0,0 +1,60 @@ +import { AIMessage, ToolMessage, type BaseMessage } from "@langchain/core/messages"; + +type Part = Record; + +export interface SemconvMessage { + role: string; + parts: Part[]; + finish_reason?: string; +} + +const ROLES: Record = { human: "user", ai: "assistant", tool: "tool" }; + +function contentParts(content: BaseMessage["content"]): Part[] { + if (typeof content === "string") return content ? [{ type: "text", content }] : []; + return content.map((block) => { + const b = block as { type?: string; text?: string }; + // Non-text blocks (images, files) are named, never inlined: they can be megabytes of base64. + return b.type === "text" + ? { type: "text", content: b.text ?? "" } + : { type: b.type ?? "unknown" }; + }); +} + +function toParts(m: BaseMessage): Part[] { + const parts = contentParts(m.content); + if (AIMessage.isInstance(m)) { + for (const call of m.tool_calls ?? []) { + parts.push({ type: "tool_call", id: call.id, name: call.name, arguments: call.args }); + } + } + if (ToolMessage.isInstance(m)) { + return [{ type: "tool_call_response", id: m.tool_call_id, response: m.content }]; + } + return parts; +} + +/** Splits a chat model's input into semconv system instructions and messages. */ +export function toSemconvInput(messages: BaseMessage[]): { + systemInstructions: Part[]; + messages: SemconvMessage[]; +} { + const systemInstructions: Part[] = []; + const out: SemconvMessage[] = []; + for (const m of messages) { + const type = m.getType(); + if (type === "system") systemInstructions.push(...contentParts(m.content)); + else out.push({ role: ROLES[type] ?? type, parts: toParts(m) }); + } + return { systemInstructions, messages: out }; +} + +export function toSemconvOutput(message: BaseMessage, finishReason?: string): SemconvMessage[] { + return [ + { + role: "assistant", + parts: toParts(message), + ...(finishReason ? { finish_reason: finishReason } : {}), + }, + ]; +} diff --git a/src/lib/tracing/semconv.ts b/src/lib/tracing/semconv.ts new file mode 100644 index 0000000..4f3b770 --- /dev/null +++ b/src/lib/tracing/semconv.ts @@ -0,0 +1,52 @@ +// OTel GenAI semantic conventions (Development stability) — the only place these names live. +// Verified against Phoenix 20.23 ingest; see docs/OBSERVABILITY.md before renaming any. + +export const GEN_AI = { + OPERATION_NAME: "gen_ai.operation.name", + PROVIDER_NAME: "gen_ai.provider.name", + AGENT_NAME: "gen_ai.agent.name", + CONVERSATION_ID: "gen_ai.conversation.id", + REQUEST_MODEL: "gen_ai.request.model", + RESPONSE_MODEL: "gen_ai.response.model", + RESPONSE_FINISH_REASONS: "gen_ai.response.finish_reasons", + USAGE_INPUT_TOKENS: "gen_ai.usage.input_tokens", + USAGE_OUTPUT_TOKENS: "gen_ai.usage.output_tokens", + SYSTEM_INSTRUCTIONS: "gen_ai.system_instructions", + INPUT_MESSAGES: "gen_ai.input.messages", + OUTPUT_MESSAGES: "gen_ai.output.messages", + TOOL_NAME: "gen_ai.tool.name", + TOOL_CALL_ID: "gen_ai.tool.call.id", + TOOL_TYPE: "gen_ai.tool.type", + TOOL_CALL_ARGUMENTS: "gen_ai.tool.call.arguments", + TOOL_CALL_RESULT: "gen_ai.tool.call.result", +} as const; + +export const OPERATION = { + INVOKE_AGENT: "invoke_agent", + CHAT: "chat", + EXECUTE_TOOL: "execute_tool", +} as const; + +export const ERROR_TYPE = "error.type"; + +export const CAMERON = { + RUN_RESUMED: "cameron.run.resumed", + APPROVALS_APPROVED: "cameron.approval.approved", + APPROVALS_REJECTED: "cameron.approval.rejected", + TOOL_MUTATING: "cameron.tool.mutating", + SPAN_UNFINISHED: "cameron.span.unfinished", + APPROVAL_REQUESTED_EVENT: "cameron.approval.requested", + APPROVAL_TOOLS: "cameron.approval.tools", +} as const; + +export const AGENT_NAME = "cameron"; + +const PROVIDER_NAMES: Record = { + google_genai: "gcp.gemini", + google_vertexai: "gcp.vertex_ai", +}; + +/** LangChain's `ls_provider` → the semconv `gen_ai.provider.name` value. */ +export function toProviderName(lsProvider: string): string { + return PROVIDER_NAMES[lsProvider] ?? lsProvider; +} diff --git a/src/services/agentService.ts b/src/services/agentService.ts index f74c0cc..a829fc4 100644 --- a/src/services/agentService.ts +++ b/src/services/agentService.ts @@ -10,10 +10,8 @@ import { readPendingBatch, resolveDecisions } from "./approvalDecisions"; import { processAttachmentsForAI } from "@/lib/storage/content"; import { streamMessages, type AgentRun } from "./messageStream"; import { projectHistory } from "./historyProjection"; -import { CallbackHandler } from "@langfuse/langchain"; - -// Only instantiate when tracing is enabled; avoids errors when Langfuse credentials are absent. -const langfuseHandler = process.env.LANGFUSE_ENABLED === "true" ? new CallbackHandler() : null; +import { GenAiTracer } from "@/lib/tracing/genAiTracer"; +import { tracingEnabled } from "@/lib/tracing/config"; /** Stream a turn (or resume a paused one) as MessageResponse chunks for the SSE route. */ export async function streamResponse(params: { @@ -36,7 +34,7 @@ export async function streamResponse(params: { const config = { version: "v3" as const, configurable: { thread_id: threadId }, - ...(langfuseHandler ? { callbacks: [langfuseHandler] } : {}), + ...(tracingEnabled() ? { callbacks: [new GenAiTracer()] } : {}), }; // A pending approval resumes the interrupt; otherwise start a fresh turn from the user message. From 73d488a19fa09bd81bda002e3c35afe46f91c88a Mon Sep 17 00:00:00 2001 From: Ali Ibrahim Jr <48456829+IBJunior@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:50:54 +0200 Subject: [PATCH 2/5] fix(tracing): keep Node-only OTel setup out of the Edge build Next.js analyzes instrumentation.ts for the Edge runtime as well, so process.once() there warned in the production build even behind the NEXT_RUNTIME check. The setup moves to instrumentation.node.ts, imported only on Node. Co-Authored-By: Claude Opus 5.5 (1M context) --- instrumentation.node.ts | 16 ++++++++++++++++ instrumentation.ts | 21 +++++---------------- 2 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 instrumentation.node.ts diff --git a/instrumentation.node.ts b/instrumentation.node.ts new file mode 100644 index 0000000..14342e6 --- /dev/null +++ b/instrumentation.node.ts @@ -0,0 +1,16 @@ +import { tracingEnabled } from "./src/lib/tracing/config"; + +export async function startTracing() { + if (!tracingEnabled()) return; + + const { NodeSDK } = await import("@opentelemetry/sdk-node"); + const { OTLPTraceExporter } = await import("@opentelemetry/exporter-trace-otlp-proto"); + + // Endpoint and headers come from the standard OTEL_EXPORTER_OTLP_* variables. + const sdk = new NodeSDK({ + serviceName: process.env.OTEL_SERVICE_NAME ?? "cameron", + traceExporter: new OTLPTraceExporter(), + }); + sdk.start(); + process.once("SIGTERM", () => void sdk.shutdown()); +} diff --git a/instrumentation.ts b/instrumentation.ts index d09e424..7f58edd 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,18 +1,7 @@ export async function register() { - // @opentelemetry/sdk-node is Node-only; Next.js also evaluates this file for the Edge runtime. - if (process.env.NEXT_RUNTIME !== "nodejs") return; - - const { tracingEnabled } = await import("./src/lib/tracing/config"); - if (!tracingEnabled()) return; - - const { NodeSDK } = await import("@opentelemetry/sdk-node"); - const { OTLPTraceExporter } = await import("@opentelemetry/exporter-trace-otlp-proto"); - - // Endpoint and headers come from the standard OTEL_EXPORTER_OTLP_* variables. - const sdk = new NodeSDK({ - serviceName: process.env.OTEL_SERVICE_NAME ?? "cameron", - traceExporter: new OTLPTraceExporter(), - }); - sdk.start(); - process.once("SIGTERM", () => void sdk.shutdown()); + // Node-only setup lives in its own file so the Edge build never analyzes it. + if (process.env.NEXT_RUNTIME === "nodejs") { + const { startTracing } = await import("./instrumentation.node"); + await startTracing(); + } } From 9dfacbf6d9f30d19c07d8fdb06078e0ea274e50a Mon Sep 17 00:00:00 2001 From: Ali Ibrahim Jr <48456829+IBJunior@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:00:04 +0200 Subject: [PATCH 3/5] fix(tracing): move instrumentation into src/ so production traces With an src/app layout, `next build` only detects instrumentation.ts inside src/. At the repo root it still ran under `pnpm dev`, but the standalone build left the hook out entirely, so the production image never started the SDK. Verified: the standalone output now contains instrumentation.js and its trace reaches Phoenix. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 6 ++++-- docs/OBSERVABILITY.md | 6 +++++- instrumentation.node.ts => src/instrumentation.node.ts | 2 +- instrumentation.ts => src/instrumentation.ts | 0 4 files changed, 10 insertions(+), 4 deletions(-) rename instrumentation.node.ts => src/instrumentation.node.ts (89%) rename instrumentation.ts => src/instrumentation.ts (100%) diff --git a/AGENTS.md b/AGENTS.md index b77eb2c..07b3b7f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -595,8 +595,10 @@ Phoenix by default. Full detail in [docs/OBSERVABILITY.md](docs/OBSERVABILITY.md gate as an event (tool names only) and the resume's approve/reject counts. - **`semconv.ts` is the only file naming `gen_ai.*` / `cameron.*` attributes.** The conventions are still Development stability; isolate any rename there. -- **`instrumentation.ts`** starts the Node SDK with the OTLP/proto exporter. Next.js adds its own - route and `fetch` spans to the same trace. +- **`src/instrumentation.ts`** starts the Node SDK with the OTLP/proto exporter. It must live in + `src/`: with an `src/app` layout the production build only looks there (a root-level file still + runs under `pnpm dev`, which hides the gap). Next.js adds its own route and `fetch` spans to the + same trace. - No vendor SDK: Langfuse, Jaeger or anything OTLP is an endpoint change, not a code change. ## OpenAPI / API Docs diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index 98ac3cf..f359c78 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -128,7 +128,7 @@ if you do. | File | Role | | -------------------------------- | ------------------------------------------------------------------------------------- | -| `instrumentation.ts` | Starts the OTel Node SDK with the OTLP/proto exporter, only when tracing is enabled | +| `src/instrumentation.ts` | Starts the OTel Node SDK with the OTLP/proto exporter, only when tracing is enabled | | `src/lib/tracing/config.ts` | `tracingEnabled()` and `captureContent()`, read from the environment | | `src/lib/tracing/genAiTracer.ts` | LangChain callback handler that turns one run's callbacks into spans; one per request | | `src/lib/tracing/semconv.ts` | The only file naming `gen_ai.*` and `cameron.*` attributes | @@ -146,6 +146,10 @@ Notes for changing it: - **The approval pause is only visible on the HITL node.** It fails with `GraphInterrupt` (which carries the gated tool names), while the run itself ends normally. A resumed run's input is the `Command`, which is where the decision counts come from. +- **`instrumentation.ts` must live in `src/`.** With an `src/app` layout, `next build` only looks + for it there; a root-level file still runs under `pnpm dev`, but the production (standalone) + build silently leaves it out, so production would never trace. Its Node-only half is + `src/instrumentation.node.ts`, imported only on the Node runtime so the Edge build never sees it. - **Model `fetch` spans are not children of the `chat` span.** Next.js creates them in the route's context; a callback handler can open spans but can't make them the active context for code it doesn't call. diff --git a/instrumentation.node.ts b/src/instrumentation.node.ts similarity index 89% rename from instrumentation.node.ts rename to src/instrumentation.node.ts index 14342e6..a67e3fd 100644 --- a/instrumentation.node.ts +++ b/src/instrumentation.node.ts @@ -1,4 +1,4 @@ -import { tracingEnabled } from "./src/lib/tracing/config"; +import { tracingEnabled } from "./lib/tracing/config"; export async function startTracing() { if (!tracingEnabled()) return; diff --git a/instrumentation.ts b/src/instrumentation.ts similarity index 100% rename from instrumentation.ts rename to src/instrumentation.ts From 846944ce6ca5bc0956a4935cc25b79951899807d Mon Sep 17 00:00:00 2001 From: Ali Ibrahim Jr <48456829+IBJunior@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:00:04 +0200 Subject: [PATCH 4/5] fix(compose): pass OPENAI_COMPATIBLE_API_KEY to the production web service Co-Authored-By: Claude Opus 5.5 (1M context) --- compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yaml b/compose.yaml index d5c8731..5e4537c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -212,6 +212,7 @@ services: ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} OPENAI_API_KEY: ${OPENAI_API_KEY:-} GOOGLE_API_KEY: ${GOOGLE_API_KEY:-} + OPENAI_COMPATIBLE_API_KEY: ${OPENAI_COMPATIBLE_API_KEY:-} # Traces go to this stack's own Phoenix, metadata only. PROD_-prefixed so a host-dev value in # .env (e.g. localhost:6006) never redirects production traces. See docs/OBSERVABILITY.md. From 5f9b62a94fd5c71b438fd990bc6372153b87d491 Mon Sep 17 00:00:00 2001 From: Ali Ibrahim Jr <48456829+IBJunior@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:08:38 +0200 Subject: [PATCH 5/5] refactor(tracing): drop deprecated LangChain message helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isBaseMessage → BaseMessage.isInstance, getType() → .type, as their deprecation notes direct. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/tracing/genAiTracer.ts | 6 ++++-- src/lib/tracing/messages.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/tracing/genAiTracer.ts b/src/lib/tracing/genAiTracer.ts index f2eb786..539e698 100644 --- a/src/lib/tracing/genAiTracer.ts +++ b/src/lib/tracing/genAiTracer.ts @@ -1,6 +1,6 @@ import { BaseCallbackHandler } from "@langchain/core/callbacks/base"; import type { Serialized } from "@langchain/core/load/serializable"; -import { AIMessage, isBaseMessage, ToolMessage, type BaseMessage } from "@langchain/core/messages"; +import { AIMessage, BaseMessage, ToolMessage } from "@langchain/core/messages"; import type { ChatGeneration, LLMResult } from "@langchain/core/outputs"; import type { ChainValues } from "@langchain/core/utils/types"; import { isGraphInterrupt } from "@langchain/langgraph"; @@ -277,7 +277,9 @@ function resumeAttributes(inputs: ChainValues): Attributes { } const messagesOf = (values: ChainValues | undefined): BaseMessage[] => - Array.isArray(values?.messages) ? values.messages.filter(isBaseMessage) : []; + Array.isArray(values?.messages) + ? values.messages.filter((m: unknown) => BaseMessage.isInstance(m)) + : []; // The turn's new messages; a resume carries a Command instead, so it has none. function runInputAttributes(inputs: ChainValues): Attributes { diff --git a/src/lib/tracing/messages.ts b/src/lib/tracing/messages.ts index 2d2f1a5..1fe3401 100644 --- a/src/lib/tracing/messages.ts +++ b/src/lib/tracing/messages.ts @@ -42,7 +42,7 @@ export function toSemconvInput(messages: BaseMessage[]): { const systemInstructions: Part[] = []; const out: SemconvMessage[] = []; for (const m of messages) { - const type = m.getType(); + const type = m.type; if (type === "system") systemInstructions.push(...contentParts(m.content)); else out.push({ role: ROLES[type] ?? type, parts: toParts(m) }); }