From 1cf1448623b7272968162556caf3e019f9473ff6 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Wed, 28 Oct 2026 15:28:39 +0530 Subject: [PATCH 01/15] feat(telemetry): Share processing across Local server and UI Extract the portable store, Node decoding pipeline, queries, and trace assembly into @sentry/telemetry. Use shared models in the Local receiver and browser workspace, and prepare independently versioned package releases. Co-Authored-By: Codex --- .craft.yml | 18 + .github/workflows/local.yml | 1 + .github/workflows/merge-jobs.yml | 17 +- .github/workflows/release.yml | 5 + apps/local/README.md | 9 + apps/local/package.json | 10 +- apps/local/src/App.integration.test.tsx | 4 +- apps/local/src/App.tsx | 5 +- .../src/components/event-detail.test.tsx | 26 +- .../investigation-controls.test.tsx | 8 +- .../components/performance-workspace.test.tsx | 10 +- .../local/src/components/span-detail.test.tsx | 6 +- .../src/components/trace-waterfall.test.tsx | 14 +- .../src/components/trace-workspace.test.tsx | 14 +- apps/local/src/lib/inspection.ts | 145 +-- apps/local/src/lib/performance.test.ts | 28 +- apps/local/src/lib/performance.ts | 176 ++- apps/local/src/lib/profiles.ts | 166 +-- .../src/lib/telemetry-regressions.test.ts | 76 ++ apps/local/src/lib/telemetry-store.test.ts | 110 +- apps/local/src/lib/telemetry-store.ts | 105 +- apps/local/src/lib/trace-model.test.ts | 43 +- apps/local/src/lib/trace-model.ts | 375 +++--- apps/local/src/test-fixtures/trace-groups.ts | 112 ++ docs/README.md | 2 + docs/cli/local-control-plane.md | 11 +- packages/cli/package.json | 6 +- .../cli/src/commands/local/control-plane.ts | 48 +- .../src/commands/local/retention-buffer.ts | 93 +- packages/cli/src/commands/local/server.ts | 40 +- .../cli/src/commands/local/session-store.ts | 17 +- .../src/commands/local/telemetry-buffer.ts | 252 ++++ packages/cli/src/lib/init/verify-setup.ts | 4 +- packages/cli/test/commands/local/run.test.ts | 6 +- .../cli/test/commands/local/server.test.ts | 52 +- .../test/commands/local/session-store.test.ts | 33 +- .../commands/local/telemetry-buffer.test.ts | 473 ++++++++ packages/telemetry/LICENSE | 13 + packages/telemetry/README.md | 205 ++++ .../continuous_profiling/profile_chunk_1.txt | 3 + .../continuous_profiling/transaction.txt | 3 + .../telemetry/_fixtures/envelope_span_v2.txt | 3 + packages/telemetry/examples/frontend.ts | 13 + packages/telemetry/examples/lifecycle.ts | 124 ++ packages/telemetry/examples/react.tsx | 28 + packages/telemetry/examples/reader.ts | 40 + packages/telemetry/examples/server.ts | 79 ++ packages/telemetry/examples/tsconfig.json | 14 + packages/telemetry/package.json | 55 + packages/telemetry/scripts/check-package.mjs | 153 +++ packages/telemetry/src/sdk/core.ts | 86 ++ packages/telemetry/src/sdk/node.spec.ts | 1052 +++++++++++++++++ packages/telemetry/src/sdk/node.ts | 645 ++++++++++ .../telemetry/src/telemetry/ai/aiLibraries.ts | 62 + packages/telemetry/src/telemetry/ai/genAI.ts | 407 +++++++ .../telemetry/src/telemetry/ai/inspection.ts | 137 +++ .../telemetry/src/telemetry/ai/vercelAISDK.ts | 536 +++++++++ .../src/telemetry/attachments.spec.ts | 87 ++ .../telemetry/src/telemetry/capabilities.ts | 18 + .../telemetry/src/telemetry/client.spec.ts | 488 ++++++++ .../src/telemetry/constants/performance.ts | 94 ++ .../src/telemetry/constants/sentry.ts | 14 + packages/telemetry/src/telemetry/contracts.ts | 87 ++ .../telemetry/src/telemetry/data/profiles.ts | 265 +++++ .../src/telemetry/envelope/codec.spec.ts | 103 ++ .../telemetry/src/telemetry/envelope/codec.ts | 171 +++ .../src/telemetry/envelope/payload.ts | 30 + .../src/telemetry/envelope/source.ts | 163 +++ .../src/telemetry/envelope/userAgent.ts | 61 + packages/telemetry/src/telemetry/index.ts | 730 ++++++++++++ .../src/telemetry/inspection-models.spec.ts | 108 ++ .../src/telemetry/internal/engine.ts | 30 + .../src/telemetry/internal/envelopes.ts | 53 + .../src/telemetry/internal/readonly.spec.ts | 30 + .../src/telemetry/internal/readonly.ts | 85 ++ .../src/telemetry/internal/results.ts | 50 + .../src/telemetry/internal/retention.ts | 16 + .../telemetry/src/telemetry/internal/types.ts | 5 + .../src/telemetry/internal/wireValues.ts | 22 + .../telemetry/src/telemetry/lifecycle.spec.ts | 463 ++++++++ .../src/telemetry/log-traces.spec.ts | 100 ++ packages/telemetry/src/telemetry/logger.ts | 23 + .../telemetry/src/telemetry/profiles/model.ts | 163 +++ packages/telemetry/src/telemetry/public.ts | 111 ++ packages/telemetry/src/telemetry/queries.ts | 2 + .../telemetry/src/telemetry/reader.spec.ts | 281 +++++ packages/telemetry/src/telemetry/reader.ts | 543 +++++++++ .../telemetry/src/telemetry/selectors/ai.ts | 18 + .../src/telemetry/selectors/database.ts | 24 + .../src/telemetry/selectors/index.ts | 14 + .../src/telemetry/selectors/profiles.ts | 48 + .../src/telemetry/selectors/resources.ts | 53 + .../src/telemetry/selectors/selectors.test.ts | 170 +++ .../src/telemetry/selectors/spans.ts | 50 + .../src/telemetry/selectors/vitalPages.ts | 58 + .../src/telemetry/selectors/webVitals.ts | 18 + .../src/telemetry/serialization.spec.ts | 257 ++++ .../telemetry/src/telemetry/serialization.ts | 218 ++++ .../telemetry/src/telemetry/store/helpers.ts | 8 + .../telemetry/store/slices/envelopesSlice.ts | 205 ++++ .../src/telemetry/store/slices/eventsSlice.ts | 330 ++++++ .../src/telemetry/store/slices/logsSlice.ts | 23 + .../telemetry/store/slices/profilesSlice.ts | 29 + .../src/telemetry/store/slices/sdksSlice.ts | 48 + .../src/telemetry/store/slices/sharedSlice.ts | 35 + .../store/slices/subscriptionsSlice.ts | 32 + .../src/telemetry/store/slices/tracesSlice.ts | 17 + .../telemetry/src/telemetry/store/types.ts | 116 ++ .../telemetry/src/telemetry/store/utils.ts | 11 + .../src/telemetry/store/utils/logProcessor.ts | 48 + .../store/utils/profileChunkProcessor.ts | 178 +++ .../telemetry/store/utils/profileProcessor.ts | 88 ++ .../telemetry/store/utils/spanProcessor.ts | 124 ++ .../telemetry/store/utils/traceInitializer.ts | 35 + .../telemetry/store/utils/traceProcessor.ts | 216 ++++ .../telemetry/src/telemetry/topology.spec.ts | 44 + packages/telemetry/src/telemetry/types.ts | 495 ++++++++ .../src/telemetry/utils/profileUtils.ts | 16 + .../src/telemetry/utils/sdkToPlatform.ts | 15 + .../telemetry/src/telemetry/utils/sentry.ts | 62 + .../telemetry/utils/stringifyValue.test.ts | 14 + .../src/telemetry/utils/stringifyValue.ts | 9 + .../telemetry/src/telemetry/utils/traces.ts | 107 ++ .../src/telemetry/utils/webVitals.ts | 200 ++++ packages/telemetry/src/telemetry/uuid.ts | 9 + packages/telemetry/tests/types/sdk-core.ts | 116 ++ packages/telemetry/tests/types/sdk-node.ts | 42 + .../tests/types/tsconfig.sdk-core.json | 13 + .../tests/types/tsconfig.sdk-node.json | 5 + packages/telemetry/tsconfig.build.json | 10 + packages/telemetry/tsconfig.json | 25 + packages/telemetry/vitest.config.ts | 9 + pnpm-lock.yaml | 298 ++--- 133 files changed, 13463 insertions(+), 1071 deletions(-) create mode 100644 apps/local/src/lib/telemetry-regressions.test.ts create mode 100644 apps/local/src/test-fixtures/trace-groups.ts create mode 100644 packages/cli/src/commands/local/telemetry-buffer.ts create mode 100644 packages/cli/test/commands/local/telemetry-buffer.test.ts create mode 100644 packages/telemetry/LICENSE create mode 100644 packages/telemetry/README.md create mode 100644 packages/telemetry/_fixtures/continuous_profiling/profile_chunk_1.txt create mode 100644 packages/telemetry/_fixtures/continuous_profiling/transaction.txt create mode 100644 packages/telemetry/_fixtures/envelope_span_v2.txt create mode 100644 packages/telemetry/examples/frontend.ts create mode 100644 packages/telemetry/examples/lifecycle.ts create mode 100644 packages/telemetry/examples/react.tsx create mode 100644 packages/telemetry/examples/reader.ts create mode 100644 packages/telemetry/examples/server.ts create mode 100644 packages/telemetry/examples/tsconfig.json create mode 100644 packages/telemetry/package.json create mode 100644 packages/telemetry/scripts/check-package.mjs create mode 100644 packages/telemetry/src/sdk/core.ts create mode 100644 packages/telemetry/src/sdk/node.spec.ts create mode 100644 packages/telemetry/src/sdk/node.ts create mode 100644 packages/telemetry/src/telemetry/ai/aiLibraries.ts create mode 100644 packages/telemetry/src/telemetry/ai/genAI.ts create mode 100644 packages/telemetry/src/telemetry/ai/inspection.ts create mode 100644 packages/telemetry/src/telemetry/ai/vercelAISDK.ts create mode 100644 packages/telemetry/src/telemetry/attachments.spec.ts create mode 100644 packages/telemetry/src/telemetry/capabilities.ts create mode 100644 packages/telemetry/src/telemetry/client.spec.ts create mode 100644 packages/telemetry/src/telemetry/constants/performance.ts create mode 100644 packages/telemetry/src/telemetry/constants/sentry.ts create mode 100644 packages/telemetry/src/telemetry/contracts.ts create mode 100644 packages/telemetry/src/telemetry/data/profiles.ts create mode 100644 packages/telemetry/src/telemetry/envelope/codec.spec.ts create mode 100644 packages/telemetry/src/telemetry/envelope/codec.ts create mode 100644 packages/telemetry/src/telemetry/envelope/payload.ts create mode 100644 packages/telemetry/src/telemetry/envelope/source.ts create mode 100644 packages/telemetry/src/telemetry/envelope/userAgent.ts create mode 100644 packages/telemetry/src/telemetry/index.ts create mode 100644 packages/telemetry/src/telemetry/inspection-models.spec.ts create mode 100644 packages/telemetry/src/telemetry/internal/engine.ts create mode 100644 packages/telemetry/src/telemetry/internal/envelopes.ts create mode 100644 packages/telemetry/src/telemetry/internal/readonly.spec.ts create mode 100644 packages/telemetry/src/telemetry/internal/readonly.ts create mode 100644 packages/telemetry/src/telemetry/internal/results.ts create mode 100644 packages/telemetry/src/telemetry/internal/retention.ts create mode 100644 packages/telemetry/src/telemetry/internal/types.ts create mode 100644 packages/telemetry/src/telemetry/internal/wireValues.ts create mode 100644 packages/telemetry/src/telemetry/lifecycle.spec.ts create mode 100644 packages/telemetry/src/telemetry/log-traces.spec.ts create mode 100644 packages/telemetry/src/telemetry/logger.ts create mode 100644 packages/telemetry/src/telemetry/profiles/model.ts create mode 100644 packages/telemetry/src/telemetry/public.ts create mode 100644 packages/telemetry/src/telemetry/queries.ts create mode 100644 packages/telemetry/src/telemetry/reader.spec.ts create mode 100644 packages/telemetry/src/telemetry/reader.ts create mode 100644 packages/telemetry/src/telemetry/selectors/ai.ts create mode 100644 packages/telemetry/src/telemetry/selectors/database.ts create mode 100644 packages/telemetry/src/telemetry/selectors/index.ts create mode 100644 packages/telemetry/src/telemetry/selectors/profiles.ts create mode 100644 packages/telemetry/src/telemetry/selectors/resources.ts create mode 100644 packages/telemetry/src/telemetry/selectors/selectors.test.ts create mode 100644 packages/telemetry/src/telemetry/selectors/spans.ts create mode 100644 packages/telemetry/src/telemetry/selectors/vitalPages.ts create mode 100644 packages/telemetry/src/telemetry/selectors/webVitals.ts create mode 100644 packages/telemetry/src/telemetry/serialization.spec.ts create mode 100644 packages/telemetry/src/telemetry/serialization.ts create mode 100644 packages/telemetry/src/telemetry/store/helpers.ts create mode 100644 packages/telemetry/src/telemetry/store/slices/envelopesSlice.ts create mode 100644 packages/telemetry/src/telemetry/store/slices/eventsSlice.ts create mode 100644 packages/telemetry/src/telemetry/store/slices/logsSlice.ts create mode 100644 packages/telemetry/src/telemetry/store/slices/profilesSlice.ts create mode 100644 packages/telemetry/src/telemetry/store/slices/sdksSlice.ts create mode 100644 packages/telemetry/src/telemetry/store/slices/sharedSlice.ts create mode 100644 packages/telemetry/src/telemetry/store/slices/subscriptionsSlice.ts create mode 100644 packages/telemetry/src/telemetry/store/slices/tracesSlice.ts create mode 100644 packages/telemetry/src/telemetry/store/types.ts create mode 100644 packages/telemetry/src/telemetry/store/utils.ts create mode 100644 packages/telemetry/src/telemetry/store/utils/logProcessor.ts create mode 100644 packages/telemetry/src/telemetry/store/utils/profileChunkProcessor.ts create mode 100644 packages/telemetry/src/telemetry/store/utils/profileProcessor.ts create mode 100644 packages/telemetry/src/telemetry/store/utils/spanProcessor.ts create mode 100644 packages/telemetry/src/telemetry/store/utils/traceInitializer.ts create mode 100644 packages/telemetry/src/telemetry/store/utils/traceProcessor.ts create mode 100644 packages/telemetry/src/telemetry/topology.spec.ts create mode 100644 packages/telemetry/src/telemetry/types.ts create mode 100644 packages/telemetry/src/telemetry/utils/profileUtils.ts create mode 100644 packages/telemetry/src/telemetry/utils/sdkToPlatform.ts create mode 100644 packages/telemetry/src/telemetry/utils/sentry.ts create mode 100644 packages/telemetry/src/telemetry/utils/stringifyValue.test.ts create mode 100644 packages/telemetry/src/telemetry/utils/stringifyValue.ts create mode 100644 packages/telemetry/src/telemetry/utils/traces.ts create mode 100644 packages/telemetry/src/telemetry/utils/webVitals.ts create mode 100644 packages/telemetry/src/telemetry/uuid.ts create mode 100644 packages/telemetry/tests/types/sdk-core.ts create mode 100644 packages/telemetry/tests/types/sdk-node.ts create mode 100644 packages/telemetry/tests/types/tsconfig.sdk-core.json create mode 100644 packages/telemetry/tests/types/tsconfig.sdk-node.json create mode 100644 packages/telemetry/tsconfig.build.json create mode 100644 packages/telemetry/tsconfig.json create mode 100644 packages/telemetry/vitest.config.ts diff --git a/.craft.yml b/.craft.yml index 6b927495d..1261f2cc0 100644 --- a/.craft.yml +++ b/.craft.yml @@ -110,3 +110,21 @@ workspaces: - name: npm id: "@sentry/mcp-server" includeNames: '/^sentry-mcp-server-\d.*\.tgz$/' + + packages/telemetry: + changelogPolicy: none + preReleaseCommand: bash -c 'cd packages/telemetry && npm version "$CRAFT_NEW_VERSION" --no-git-tag-version' + releaseBranchPrefix: release/telemetry + artifactProvider: + name: github + config: + artifacts: + Post-merge tasks: + - "/^sentry-telemetry-.*$/" + targets: + - name: github + tagPrefix: "telemetry@" + includeNames: '/^sentry-telemetry-\d.*\.tgz$/' + - name: npm + id: "@sentry/telemetry" + includeNames: '/^sentry-telemetry-\d.*\.tgz$/' diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml index c287154ad..ae0cacbee 100644 --- a/.github/workflows/local.yml +++ b/.github/workflows/local.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - "apps/local/**" + - "packages/telemetry/**" - "packages/cli/src/commands/local/**" - "packages/cli/test/commands/local/**" - "packages/cli/test/e2e/local.test.ts" diff --git a/.github/workflows/merge-jobs.yml b/.github/workflows/merge-jobs.yml index c021908dc..d1c1f067b 100644 --- a/.github/workflows/merge-jobs.yml +++ b/.github/workflows/merge-jobs.yml @@ -2,7 +2,7 @@ name: Post-merge tasks on: push: - branches: ["main", "release/*", "release/mcp/**"] + branches: ["main", "release/*", "release/mcp/**", "release/telemetry/**"] workflow_dispatch: jobs: @@ -38,6 +38,14 @@ jobs: working-directory: packages/mcp-server run: pnpm pack --pack-destination dist + - name: Package telemetry + working-directory: packages/telemetry + run: pnpm pack --pack-destination dist + + - name: Check packed telemetry consumer + working-directory: packages/telemetry + run: node scripts/check-package.mjs dist/sentry-telemetry-*.tgz + - name: Archive artifacts uses: actions/upload-artifact@v4 with: @@ -45,6 +53,13 @@ jobs: path: | ${{ github.workspace }}/packages/mcp-server/dist/*.tgz + - name: Archive telemetry artifact + uses: actions/upload-artifact@v4 + with: + name: sentry-telemetry-${{ github.sha }} + path: ${{ github.workspace }}/packages/telemetry/dist/*.tgz + if-no-files-found: error + - name: Publish Test Report uses: mikepenz/action-junit-report@cf701569b05ccdd861a76b8607a66d76f6fd4857 if: ${{ !cancelled() }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8eeea45b0..d9b48f0f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,7 @@ on: options: - cli - mcp + - telemetry bump: description: Version bump type required: true @@ -68,6 +69,10 @@ jobs: workspace=packages/mcp-server package_path=packages/mcp-server ;; + telemetry) + workspace=packages/telemetry + package_path=packages/telemetry + ;; *) echo "::error::Unsupported product: $PRODUCT" exit 1 diff --git a/apps/local/README.md b/apps/local/README.md index 2a362420f..a9f456b99 100644 --- a/apps/local/README.md +++ b/apps/local/README.md @@ -13,6 +13,15 @@ The receiver can also be a named Local session or an explicitly selected remote receiver. Capture lifecycle and retention are managed through the existing Local controls. +The browser uses `@sentry/telemetry` for trace assembly, database and resource +summaries, AI field interpretation, profile call trees and Web Vitals analysis. +Raw envelopes, item headers and attachment payloads remain available for inspection. +Each receiver connection owns a bounded, memory-only processing store; clearing +or switching captures also invalidates pending processing. SDK timestamps use +milliseconds; the Local waterfall adapter converts its display coordinates to +wire seconds. Transactions without trace context remain visible as standalone +navigation groups. + ## Replay example captures With a receiver running, load all bundled examples: diff --git a/apps/local/package.json b/apps/local/package.json index eab1c58ea..dbfb5e9c3 100644 --- a/apps/local/package.json +++ b/apps/local/package.json @@ -4,19 +4,20 @@ "version": "0.0.0", "type": "module", "scripts": { - "dev": "vite", - "build": "tsc -b && vite build", + "dev": "pnpm --filter @sentry/telemetry run build && vite", + "build": "pnpm --filter @sentry/telemetry run build && tsc -b && vite build", "lint": "oxlint", "preview": "vite preview", - "test": "vitest run", + "test": "pnpm --filter @sentry/telemetry run build && vitest run", "test:integration": "vitest run src/App.integration.test.tsx", - "typecheck": "tsc -b", + "typecheck": "pnpm --filter @sentry/telemetry run build && tsc -b", "sample": "tsx scripts/replay.ts" }, "engines": { "node": "^20.19.0 || >=22.12.0" }, "dependencies": { + "@sentry/telemetry": "workspace:*", "@hookform/resolvers": "^5.9.1", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-popover": "^1.3.1", @@ -45,7 +46,6 @@ "valibot": "catalog:" }, "devDependencies": { - "@spotlightjs/spotlight": "^4.11.7", "@testing-library/react": "^16.3.2", "@types/node": "^24.13.3", "@types/react": "^19.2.18", diff --git a/apps/local/src/App.integration.test.tsx b/apps/local/src/App.integration.test.tsx index 1bc432db5..3857b07da 100644 --- a/apps/local/src/App.integration.test.tsx +++ b/apps/local/src/App.integration.test.tsx @@ -1,5 +1,5 @@ import { replayEnvelopes, defaultFixtures } from "../scripts/replay.ts"; -import { createSpotlightBuffer } from "@spotlightjs/spotlight/sdk"; +import { createLocalTelemetryBuffer } from "../../../packages/cli/src/commands/local/telemetry-buffer.ts"; import { EventSource as NodeEventSource } from "eventsource"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { @@ -75,7 +75,7 @@ function envelope(transaction: string, options: EnvelopeOptions = {}): string { async function startReceiver() { return await tryListen( - buildApp(createSpotlightBuffer(10), { uiActions: true }), + buildApp(createLocalTelemetryBuffer(10), { uiActions: true }), 0, "127.0.0.1", ); diff --git a/apps/local/src/App.tsx b/apps/local/src/App.tsx index 196e30e9d..39726f745 100644 --- a/apps/local/src/App.tsx +++ b/apps/local/src/App.tsx @@ -45,7 +45,6 @@ import { Menu, Settings, Search } from "lucide-react"; import { useCallback, useEffect, - useMemo, useRef, useState, useSyncExternalStore, @@ -95,7 +94,7 @@ import { type LocalFeedItem, } from "@/lib/spotlight.ts"; import { createLocalTelemetryStore } from "@/lib/telemetry-store.ts"; -import { buildTraceGroups, traceIdForItem } from "@/lib/trace-model.ts"; +import { traceIdForItem } from "@/lib/trace-model.ts"; import { workspaceNavigation } from "@/lib/workspace-navigation.ts"; import { isErrorEvent, @@ -403,7 +402,7 @@ export default function App() { () => saveConnectionHistory(recentConnections), [recentConnections], ); - const traces = useMemo(() => buildTraceGroups(items), [items]); + const traces = telemetry.traceGroups; const searchQuery = workspaceQuery.query; const filterKey = `${workspaceView}:${streamUrl}`; const [eventFilterState, setEventFilterState] = useState<{ diff --git a/apps/local/src/components/event-detail.test.tsx b/apps/local/src/components/event-detail.test.tsx index 2cb389db4..ce8ee8a6b 100644 --- a/apps/local/src/components/event-detail.test.tsx +++ b/apps/local/src/components/event-detail.test.tsx @@ -1,7 +1,7 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { afterEach, expect, test, vi } from "vitest"; import { decodeEnvelope } from "@/lib/spotlight.ts"; -import { buildTraceGroups } from "@/lib/trace-model.ts"; +import { buildTraceGroups } from "@/test-fixtures/trace-groups.ts"; import { EventDetail } from "./event-detail.tsx"; afterEach(() => { @@ -17,7 +17,7 @@ function item(type: string, payload: unknown) { return entry; } -test("logs display their message, severity, and structured attributes", () => { +test("logs display their message, severity, and structured attributes", async () => { render( { expect(screen.getByRole("button", { name: "Copy Message" })).toBeDefined(); }); -test("raw-only SDK payloads default to JSON and reset on a different item", () => { +test("raw-only SDK payloads default to JSON and reset on a different item", async () => { const { rerender } = render( , ); @@ -61,7 +61,7 @@ test("raw-only SDK payloads default to JSON and reset on a different item", () = ).toBe("true"); }); -test("arrow navigation wraps between detail tabs and moves keyboard focus", () => { +test("arrow navigation wraps between detail tabs and moves keyboard focus", async () => { render(); const overview = screen.getByRole("tab", { name: "Overview" }); fireEvent.keyDown(overview, { key: "ArrowLeft" }); @@ -73,7 +73,7 @@ test("arrow navigation wraps between detail tabs and moves keyboard focus", () = expect(document.activeElement).toBe(overview); }); -test("raw envelopes retain their recognizable heading and open directly in JSON", () => { +test("raw envelopes retain their recognizable heading and open directly in JSON", async () => { const envelope = { id: "envelope:1", type: "envelope", @@ -91,7 +91,7 @@ test("raw envelopes retain their recognizable heading and open directly in JSON" ); }); -test("malformed and nested log attributes remain readable inert text", () => { +test("malformed and nested log attributes remain readable inert text", async () => { render( { expect(attributes.querySelector("script")).toBeNull(); }); -test("requests show a compact copyable summary with captured method and URL", () => { +test("requests show a compact copyable summary with captured method and URL", async () => { const copied: string[] = []; vi.stubGlobal("navigator", { clipboard: { @@ -155,7 +155,7 @@ test("requests show a compact copyable summary with captured method and URL", () expect(copied).toEqual(["201"]); }); -test("span summaries use captured typed URL attributes without inventing request fields", () => { +test("span summaries use captured typed URL attributes without inventing request fields", async () => { render( { +test("repeated parent updates keep one request summary and its waterfall", async () => { const transaction = item("transaction", { transaction: "GET /orders", start_timestamp: 1, @@ -184,7 +184,7 @@ test("repeated parent updates keep one request summary and its waterfall", () => request: { status_code: 200 }, contexts: { trace: { trace_id: "trace", span_id: "root" } }, }); - const [trace] = buildTraceGroups([transaction]); + const [trace] = await buildTraceGroups([transaction]); const { rerender } = render(); for (let index = 0; index < 10; index += 1) { rerender( @@ -204,7 +204,7 @@ test("repeated parent updates keep one request summary and its waterfall", () => ).toBeDefined(); }); -test("captured context suppresses the empty-details message", () => { +test("captured context suppresses the empty-details message", async () => { render( { expect(screen.queryByText("No additional event details.")).toBeNull(); }); -test("a related trace suppresses the empty-details message", () => { +test("a related trace suppresses the empty-details message", async () => { const transaction = item("transaction", { transaction: "GET /trace", contexts: { trace: { trace_id: "trace", span_id: "root" } }, start_timestamp: 1, timestamp: 2, }); - const [trace] = buildTraceGroups([transaction]); + const [trace] = await buildTraceGroups([transaction]); render(); expect(screen.getByRole("region", { name: "Trace waterfall" })).toBeDefined(); expect(screen.queryByText("No additional event details.")).toBeNull(); diff --git a/apps/local/src/components/investigation-controls.test.tsx b/apps/local/src/components/investigation-controls.test.tsx index 1c27a917b..84bf0c1ff 100644 --- a/apps/local/src/components/investigation-controls.test.tsx +++ b/apps/local/src/components/investigation-controls.test.tsx @@ -1,7 +1,7 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { afterEach, expect, test } from "vitest"; import { decodeEnvelope } from "@/lib/spotlight.ts"; -import { buildTraceGroups } from "@/lib/trace-model.ts"; +import { buildTraceGroups } from "@/test-fixtures/trace-groups.ts"; import { TraceWaterfall } from "./trace-waterfall.tsx"; import { filterEvents, defaultEventFilters } from "@/lib/event-filters.ts"; afterEach(cleanup); @@ -32,7 +32,7 @@ const events = decodeEnvelope( ]), "logs", ); -test("log filters combine severity and logger and sort timestamps without changing captured data", () => { +test("log filters combine severity and logger and sort timestamps without changing captured data", async () => { expect( filterEvents(events, { ...defaultEventFilters, @@ -47,8 +47,8 @@ test("log filters combine severity and logger and sort timestamps without changi ).toBe(20); expect(events[0]?.timestamp).toBe(10); }); -test("trace search reveals matching descendants and branch collapse is reversible", () => { - const [trace] = buildTraceGroups( +test("trace search reveals matching descendants and branch collapse is reversible", async () => { + const [trace] = await buildTraceGroups( decodeEnvelope( JSON.stringify([ {}, diff --git a/apps/local/src/components/performance-workspace.test.tsx b/apps/local/src/components/performance-workspace.test.tsx index 150ad788d..6e6a7e0ac 100644 --- a/apps/local/src/components/performance-workspace.test.tsx +++ b/apps/local/src/components/performance-workspace.test.tsx @@ -1,14 +1,14 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { afterEach, expect, test, vi } from "vitest"; import { decodeEnvelope } from "@/lib/spotlight.ts"; -import { buildTraceGroups } from "@/lib/trace-model.ts"; +import { buildTraceGroups } from "@/test-fixtures/trace-groups.ts"; import { SpanAnalysis, WebVitalsWorkspace } from "./performance-workspace.tsx"; import { ProfileWorkspace } from "./profile-workspace.tsx"; afterEach(cleanup); const item = (type: string, payload: unknown) => decodeEnvelope(JSON.stringify([{}, [[{ type }, payload]]]), "sample")[0]!; -test("resource groups sort and open the selected execution in its trace", () => { - const traces = buildTraceGroups([ +test("resource groups sort and open the selected execution in its trace", async () => { + const traces = await buildTraceGroups([ item("transaction", { transaction: "Page", contexts: { trace: { trace_id: "t", span_id: "root" } }, @@ -47,7 +47,7 @@ test("resource groups sort and open the selected execution in its trace", () => fireEvent.click(screen.getByRole("button", { name: /Page.*two.*100.0 ms/ })); expect(onSelect).toHaveBeenCalledWith("t", "two"); }); -test("Web Vitals metric switching displays captured INP without fabricating missing visits", () => { +test("Web Vitals metric switching displays captured INP without fabricating missing visits", async () => { render( { +test("profile chart supports zoom, visualization switching and independent thread samples", async () => { const profile = item("profile", { profile: { frames: [{ function: "mainWork" }, { function: "workerWork" }], diff --git a/apps/local/src/components/span-detail.test.tsx b/apps/local/src/components/span-detail.test.tsx index 00ec3cade..05ebb8039 100644 --- a/apps/local/src/components/span-detail.test.tsx +++ b/apps/local/src/components/span-detail.test.tsx @@ -1,13 +1,13 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { afterEach, expect, test, vi } from "vitest"; import { decodeEnvelope } from "@/lib/spotlight.ts"; -import { buildTraceGroups } from "@/lib/trace-model.ts"; +import { buildTraceGroups } from "@/test-fixtures/trace-groups.ts"; import { SpanDetail } from "./span-detail.tsx"; afterEach(cleanup); -test("span inspection shows its own timing, attributes and JSON plus navigable trace telemetry", () => { - const [trace] = buildTraceGroups( +test("span inspection shows its own timing, attributes and JSON plus navigable trace telemetry", async () => { + const [trace] = await buildTraceGroups( decodeEnvelope( JSON.stringify([ {}, diff --git a/apps/local/src/components/trace-waterfall.test.tsx b/apps/local/src/components/trace-waterfall.test.tsx index 8aea70b97..ac2a301dd 100644 --- a/apps/local/src/components/trace-waterfall.test.tsx +++ b/apps/local/src/components/trace-waterfall.test.tsx @@ -2,14 +2,14 @@ import { afterEach, expect, test } from "vitest"; import { renderToStaticMarkup } from "react-dom/server"; import { cleanup, render, screen } from "@testing-library/react"; import { decodeEnvelope } from "@/lib/spotlight.ts"; -import { buildTraceGroups } from "@/lib/trace-model.ts"; +import { buildTraceGroups } from "@/test-fixtures/trace-groups.ts"; import { TraceWaterfall } from "./trace-waterfall.tsx"; afterEach(cleanup); test.each(["internal_error", "error"])( "styles status-only %s failures without affecting healthy children", - (status) => { + async (status) => { const items = decodeEnvelope( JSON.stringify([ {}, @@ -47,7 +47,7 @@ test.each(["internal_error", "error"])( ]), "envelope", ); - const trace = buildTraceGroups(items)[0]; + const trace = (await buildTraceGroups(items))[0]; expect(trace).toBeDefined(); if (!trace) throw new Error("Expected trace"); render(); @@ -63,7 +63,7 @@ test.each(["internal_error", "error"])( }, ); -test("renders a deeply nested valid span chain without overflowing the stack", () => { +test("renders a deeply nested valid span chain without overflowing the stack", async () => { const count = 4000; const items = decodeEnvelope( JSON.stringify([ @@ -89,7 +89,7 @@ test("renders a deeply nested valid span chain without overflowing the stack", ( ]), "deep-envelope", ); - const trace = buildTraceGroups(items)[0]; + const trace = (await buildTraceGroups(items))[0]; if (!trace) throw new Error("Expected deep trace"); const markup = renderToStaticMarkup(); expect(markup).toContain(`waterfall-bar-span-${count - 1}`); @@ -98,8 +98,8 @@ test("renders a deeply nested valid span chain without overflowing the stack", ( ); }); -test("span descriptions are keyboard-selectable and expose selected state", () => { - const [trace] = buildTraceGroups( +test("span descriptions are keyboard-selectable and expose selected state", async () => { + const [trace] = await buildTraceGroups( decodeEnvelope( JSON.stringify([ {}, diff --git a/apps/local/src/components/trace-workspace.test.tsx b/apps/local/src/components/trace-workspace.test.tsx index 83d42ff99..55aa14a92 100644 --- a/apps/local/src/components/trace-workspace.test.tsx +++ b/apps/local/src/components/trace-workspace.test.tsx @@ -2,13 +2,13 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { useState } from "react"; import { afterEach, expect, test, vi } from "vitest"; import { decodeEnvelope } from "@/lib/spotlight.ts"; -import { buildTraceGroups } from "@/lib/trace-model.ts"; +import { buildTraceGroups } from "@/test-fixtures/trace-groups.ts"; import { TraceWorkspace } from "./trace-workspace.tsx"; afterEach(cleanup); -test("selects spans within the trace inspector and returns to the waterfall", () => { - const traces = buildTraceGroups( +test("selects spans within the trace inspector and returns to the waterfall", async () => { + const traces = await buildTraceGroups( decodeEnvelope( JSON.stringify([ {}, @@ -56,8 +56,8 @@ test("selects spans within the trace inspector and returns to the waterfall", () expect(clearSpan).toHaveBeenCalledOnce(); }); -test("a stale trace selection cannot drill into a duplicate span ID on a fallback trace", () => { - const traces = buildTraceGroups( +test("a stale trace selection cannot drill into a duplicate span ID on a fallback trace", async () => { + const traces = await buildTraceGroups( decodeEnvelope( JSON.stringify([ {}, @@ -94,8 +94,8 @@ test("a stale trace selection cannot drill into a duplicate span ID on a fallbac ).toBe("false"); }); -test("drilling into a span and returning restores keyboard focus", () => { - const traces = buildTraceGroups( +test("drilling into a span and returning restores keyboard focus", async () => { + const traces = await buildTraceGroups( decodeEnvelope( JSON.stringify([ {}, diff --git a/apps/local/src/lib/inspection.ts b/apps/local/src/lib/inspection.ts index 27431074c..f59f3fa44 100644 --- a/apps/local/src/lib/inspection.ts +++ b/apps/local/src/lib/inspection.ts @@ -1,3 +1,8 @@ +import { inspectAI, isAIInteraction as isAiPayload } from "@sentry/telemetry"; +export { + isAIInteraction as isAiPayload, + readSpanData as spanData, +} from "@sentry/telemetry"; import { getEventMetadata, type LocalFeedItem } from "./spotlight.ts"; import { getEventExceptions, formatExceptionTitle } from "./exceptions.ts"; @@ -27,39 +32,6 @@ export function parsed(value: unknown): unknown { return value; } } -/** Merge legacy span data and typed span-v2 attributes without losing unknown fields. */ -export function spanData(payload: unknown): Record { - const p = record(payload); - const attributes = Object.fromEntries( - Object.entries(record(p.attributes)).map(([key, value]) => [ - key, - "value" in record(value) ? record(value).value : value, - ]), - ); - return { - ...record(record(record(p.contexts).trace).data), - ...record(p.data), - ...attributes, - }; -} -export function isAiPayload(payload: unknown): boolean { - const p = record(payload), - data = spanData(p); - return ( - [ - p.type, - p.op, - record(record(p.contexts).trace).op, - data["sentry.op"], - data["sentry.origin"], - ].some( - (value) => - typeof value === "string" && - /(?:^|[.\s:_-])(?:ai|gen_ai|llm)(?:$|[.\s:_-])/i.test(value), - ) || - Object.keys(data).some((key) => /^(?:gen_ai|vercel\.ai|ai|llm)\./.test(key)) - ); -} /** Include AI spans embedded in transactions, deduplicated against standalone copies. */ export function aiItems(items: readonly LocalFeedItem[]): LocalFeedItem[] { const found = new Map(); @@ -102,97 +74,24 @@ export type AiInspection = { tools: unknown[]; }; export function inspectAi(payload: unknown): AiInspection | undefined { - if (!isAiPayload(payload)) return; - const data = spanData(payload); - const pick = (...keys: string[]) => - keys - .map((key) => data[key]) - .find((value) => value !== undefined && value !== null); - const definitions: [string, string[]][] = [ - [ - "Model", - ["gen_ai.request.model", "gen_ai.response.model", "vercel.ai.model.id"], - ], - [ - "Provider", - ["gen_ai.system", "gen_ai.provider.name", "vercel.ai.model.provider"], - ], - [ - "Input tokens", - [ - "gen_ai.usage.input_tokens", - "vercel.ai.usage.promptTokens", - "ai.usage.promptTokens", - ], - ], - [ - "Output tokens", - [ - "gen_ai.usage.output_tokens", - "vercel.ai.usage.completionTokens", - "ai.usage.completionTokens", - ], - ], - [ - "Finish reason", - ["gen_ai.response.finish_reasons", "vercel.ai.response.finishReason"], - ], - ["Function", ["gen_ai.function_id", "vercel.ai.telemetry.functionId"]], - ["Max retries", ["vercel.ai.settings.maxRetries"]], - ["Max steps", ["vercel.ai.settings.maxSteps"]], - ]; - const calls = parsed( - pick( - "gen_ai.response.tool_calls", - "vercel.ai.response.toolCalls", - "ai.response.toolCalls", - ), - ); - const tools = Array.isArray(calls) - ? [...calls] - : calls === undefined - ? [] - : [calls]; - const toolName = pick("gen_ai.tool.name", "ai.toolCall.name"); - if (toolName !== undefined) - tools.push({ - name: toolName, - id: pick("gen_ai.tool.call.id", "ai.toolCall.id"), - arguments: parsed(pick("gen_ai.tool.input", "ai.toolCall.args")), - result: parsed(pick("gen_ai.tool.output", "ai.toolCall.result")), - }); + const ai = inspectAI(payload); + if (!ai) return; + const labels: Record = { + model: "Model", + provider: "Provider", + inputTokens: "Input tokens", + outputTokens: "Output tokens", + finishReason: "Finish reason", + function: "Function", + maxRetries: "Max retries", + maxSteps: "Max steps", + }; return { - fields: definitions.flatMap(([label, keys]) => { - const value = pick(...keys); - return value === undefined ? [] : [{ label, value: display(value) }]; - }), - system: parsed( - pick( - "gen_ai.system_instructions", - "gen_ai.prompt.system", - "ai.prompt.system", - ), - ), - messages: parsed( - pick( - "gen_ai.request.messages", - "gen_ai.input.messages", - "vercel.ai.prompt.messages", - "vercel.ai.prompt", - "gen_ai.prompt", - "ai.prompt.messages", - "ai.prompt", - ), - ), - response: parsed( - pick( - "gen_ai.response.text", - "gen_ai.output.messages", - "vercel.ai.response.text", - "ai.response.text", - ), - ), - tools, + ...ai, + fields: Object.entries(ai.fields).map(([key, value]) => ({ + label: labels[key] ?? key, + value: display(value), + })), }; } export function feedbackData(payload: unknown): Record { diff --git a/apps/local/src/lib/performance.test.ts b/apps/local/src/lib/performance.test.ts index 2e7628903..69a410e6f 100644 --- a/apps/local/src/lib/performance.test.ts +++ b/apps/local/src/lib/performance.test.ts @@ -1,11 +1,11 @@ import { expect, test } from "vitest"; import { decodeEnvelope } from "./spotlight.ts"; -import { buildTraceGroups } from "./trace-model.ts"; +import { buildTraceGroups } from "../test-fixtures/trace-groups.ts"; import { spanInsights, webVitalPages } from "./performance.ts"; import { parseProfile, profileMatchesTrace } from "./profiles.ts"; const decode = (type: string, payload: unknown) => decodeEnvelope(JSON.stringify([{}, [[{ type }, payload]]]), "fixture")[0]!; -test("query analysis groups embedded and standalone executions without counting a copied span twice", () => { +test("query analysis groups embedded and standalone executions without counting a copied span twice", async () => { const transaction = decode("transaction", { transaction: "GET /orders", start_timestamp: 1, @@ -30,7 +30,7 @@ test("query analysis groups embedded and standalone executions without counting timestamp: 1.1, }); const [group] = spanInsights( - buildTraceGroups([transaction, copy]), + await buildTraceGroups([transaction, copy]), "queries", ); expect(group).toMatchObject({ count: 1, averageMs: 100, totalMs: 100 }); @@ -39,7 +39,7 @@ test("query analysis groups embedded and standalone executions without counting span: { id: "q1" }, }); }); -test("Web Vitals retain per-visit units and classify thresholds instead of inventing missing measurements", () => { +test("Web Vitals retain per-visit units and classify thresholds instead of inventing missing measurements", async () => { const [page] = webVitalPages([ decode("transaction", { transaction: "/orders", @@ -56,7 +56,7 @@ test("Web Vitals retain per-visit units and classify thresholds instead of inven }); expect(page?.visits[0]?.metrics.inp).toBeUndefined(); }); -test("profiles build root-first trees, inclusive/self samples, and interval times per thread", () => { +test("profiles build root-first trees, inclusive/self samples, and interval times per thread", async () => { const payload = { profile: { frames: [{ function: "main" }, { function: "query" }], @@ -81,7 +81,7 @@ test("profiles build root-first trees, inclusive/self samples, and interval time }); expect(profile.threads).toHaveLength(2); }); -test("continuous chunks and missing samples remain honest", () => { +test("continuous chunks and missing samples remain honest", async () => { const chunk = parseProfile( decode("profile_chunk", { profile: { @@ -98,7 +98,7 @@ test("continuous chunks and missing samples remain honest", () => { expect(parseProfile(decode("profile", {})).sampleCount).toBe(0); }); -test("profiles require captured IDs before associating standalone spans or logs", () => { +test("profiles require captured IDs before associating standalone spans or logs", async () => { const chunk = decode("profile_chunk", { profiler_id: "unrelated", profile: { samples: [{ timestamp: 1000 }] }, @@ -112,13 +112,13 @@ test("profiles require captured IDs before associating standalone spans or logs" }), decode("log", { trace_id: "other", timestamp: 1, body: "Unrelated log" }), ]) { - expect(profileMatchesTrace(chunk, buildTraceGroups([item])[0]!)).toBe( - false, - ); + expect( + profileMatchesTrace(chunk, (await buildTraceGroups([item]))[0]!), + ).toBe(false); } }); -test("legacy and continuous profiles retain real transaction associations", () => { +test("legacy and continuous profiles retain real transaction associations", async () => { const transaction = decode("transaction", { event_id: "event", start_timestamp: 1, @@ -128,7 +128,7 @@ test("legacy and continuous profiles retain real transaction associations", () = profile: { profiler_id: "profiler", profile_id: "legacy" }, }, }); - const trace = buildTraceGroups([transaction])[0]!; + const trace = (await buildTraceGroups([transaction]))[0]!; for (const payload of [ { transaction: { id: "event" } }, { transactions: [{ trace_id: "t" }] }, @@ -147,7 +147,7 @@ test("legacy and continuous profiles retain real transaction associations", () = ).toBe(false); }); -test("large continuous profiles can be matched without exceeding argument limits", () => { +test("large continuous profiles can be matched without exceeding argument limits", async () => { const transaction = decode("transaction", { event_id: "event", start_timestamp: 1000, @@ -157,7 +157,7 @@ test("large continuous profiles can be matched without exceeding argument limits profile: { profiler_id: "profiler" }, }, }); - const trace = buildTraceGroups([transaction])[0]!; + const trace = (await buildTraceGroups([transaction]))[0]!; const profile = { samples: Array.from({ length: 150_000 }, (_, index) => ({ timestamp: 1000 + index / 100, diff --git a/apps/local/src/lib/performance.ts b/apps/local/src/lib/performance.ts index efc6256bb..09acf8a28 100644 --- a/apps/local/src/lib/performance.ts +++ b/apps/local/src/lib/performance.ts @@ -1,6 +1,12 @@ +import { + createTelemetryReader, + groupWebVitalPages, + type VitalVisit as SharedVitalVisit, + type TelemetrySnapshot, +} from "@sentry/telemetry"; import type { LocalFeedItem } from "./spotlight.ts"; import type { TraceGroup, TraceSpan } from "./trace-model.ts"; -import { finite, record, spanData, text } from "./inspection.ts"; +import { record, text } from "./inspection.ts"; export type SpanInsight = { id: string; name: string; @@ -15,101 +21,89 @@ export function spanInsights( traces: readonly TraceGroup[], kind: "queries" | "resources", ): SpanInsight[] { - const groups = new Map(); - for (const trace of traces) - for (const span of trace.spans) { - if ( - !(kind === "queries" ? /^db(?:\.|$)/ : /^resource(?:\.|$)/).test( - span.operation ?? "", - ) - ) - continue; - const id = `${span.operation}:${span.description}`; - const group = groups.get(id) ?? { - id, - name: span.description, - operation: span.operation ?? "", - count: 0, - executions: [], - }; - group.count++; - group.executions.push({ - traceId: trace.id, - traceTitle: trace.title, - span, - }); - groups.set(id, group); - } - for (const group of groups.values()) { - const durations = group.executions.flatMap((entry) => - entry.span.durationMs === undefined ? [] : [entry.span.durationMs], - ); - if (durations.length) { - group.totalMs = durations.reduce((a, b) => a + b, 0); - group.averageMs = group.totalMs / durations.length; - } - const sizes = group.executions.flatMap((entry) => { - const data = spanData(entry.span.payload); - const size = - finite(data["http.response_content_length"]) ?? - finite(data["http.response.body.size"]) ?? - finite(data["http.decoded_response_content_length"]); - return size === undefined || size < 0 ? [] : [size]; + const reader = createTelemetryReader( + () => + ({ + revision: 0, + envelopes: [], + events: [], + logs: [], + sdks: [], + profiles: new Map(), + traces: traces.flatMap(({ processed }) => + processed + ? [ + { + ...processed, + spans: new Map( + [...processed.spans].map(([id, span]) => [ + id, + { ...span, trace_id: processed.trace_id }, + ]), + ), + }, + ] + : [], + ), + }) satisfies TelemetrySnapshot, + ); + const summaries = + kind === "queries" + ? reader.getDatabaseQuerySummaries({ includeSpans: true }) + : reader.getResourceSummaries({ includeSpans: true }); + return summaries.map((summary) => { + const executions = summary.spans.flatMap((processed) => { + // Embedded legacy spans need their containing trace to resolve their ID. + const trace = traces.find((trace) => trace.id === processed.trace_id); + const span = trace?.spans.find((span) => span.id === processed.span_id); + return trace && span + ? [{ traceId: trace.id, traceTitle: trace.title, span }] + : []; }); - if (sizes.length) - group.averageBytes = sizes.reduce((a, b) => a + b, 0) / sizes.length; - } - return [...groups.values()]; + return { + id: summary.description, + name: summary.description, + operation: summary.spans[0]?.op ?? "", + count: summary.count, + totalMs: summary.totalDurationMs, + averageMs: summary.averageDurationMs, + averageBytes: + "averageSizeBytes" in summary && + typeof summary.averageSizeBytes === "number" + ? summary.averageSizeBytes + : undefined, + executions, + }; + }); } -export const VITALS = { - lcp: { label: "LCP", unit: "ms", good: 2500, poor: 4000 }, - inp: { label: "INP", unit: "ms", good: 200, poor: 500 }, - cls: { label: "CLS", unit: "", good: 0.1, poor: 0.25 }, - fcp: { label: "FCP", unit: "ms", good: 1800, poor: 3000 }, - ttfb: { label: "TTFB", unit: "ms", good: 800, poor: 1800 }, - fid: { label: "FID (legacy)", unit: "ms", good: 100, poor: 300 }, -} as const; -export type VitalName = keyof typeof VITALS; -export type VitalMeasurement = { - value: number; - rating: "Good" | "Needs improvement" | "Poor"; -}; + +export { WEB_VITALS as VITALS } from "@sentry/telemetry"; +export type { VitalName, VitalMeasurement } from "@sentry/telemetry"; export type VitalVisit = { item: LocalFeedItem; - metrics: Partial>; + metrics: SharedVitalVisit["metrics"]; }; export type VitalPage = { name: string; visits: VitalVisit[] }; export function webVitalPages(items: readonly LocalFeedItem[]): VitalPage[] { - const pages = new Map(); - for (const item of items) { - const p = record(item.payload), - measurements = record(p.measurements); - if (item.type !== "transaction" || !Object.keys(measurements).length) - continue; - const metrics: VitalVisit["metrics"] = {}; - for (const name of Object.keys(VITALS) as VitalName[]) { - const metric = record(measurements[name]), - value = finite(metric.value); - if (value === undefined || value < 0) continue; - const config = VITALS[name]; - // SDK measurements normally use milliseconds; honor seconds when explicitly supplied. - const normalized = - config.unit === "ms" && metric.unit === "second" ? value * 1000 : value; - metrics[name] = { - value: normalized, - rating: - normalized <= config.good - ? "Good" - : normalized <= config.poor - ? "Needs improvement" - : "Poor", - }; - } - if (!Object.keys(metrics).length) continue; - const name = text(p.transaction) ?? item.metadata?.route ?? "Unknown page"; - const page = pages.get(name) ?? { name, visits: [] }; - page.visits.push({ item, metrics }); - pages.set(name, page); - } - return [...pages.values()].sort((a, b) => a.name.localeCompare(b.name)); + const byId = new Map(items.map((item, index) => [String(index), item])); + const pages = groupWebVitalPages( + items.flatMap((item, index) => { + if (item.type !== "transaction") return []; + const payload = record(item.payload); + return [ + { + id: String(index), + transaction: text(payload.transaction) ?? item.metadata?.route, + measurements: payload.measurements, + }, + ]; + }), + ); + return pages.map((page) => ({ + ...page, + visits: page.visits.map((visit) => ({ + item: byId.get(visit.eventId)!, + metrics: visit.metrics, + })), + })); } diff --git a/apps/local/src/lib/profiles.ts b/apps/local/src/lib/profiles.ts index b73415b79..cfee379a7 100644 --- a/apps/local/src/lib/profiles.ts +++ b/apps/local/src/lib/profiles.ts @@ -1,168 +1,20 @@ +import { buildProfileModel, type ParsedProfile } from "@sentry/telemetry"; import type { LocalFeedItem } from "./spotlight.ts"; import type { TraceGroup } from "./trace-model.ts"; import { record, finite, text } from "./inspection.ts"; -export type ProfileNode = { - id: string; - name: string; - location: string; - inApp: boolean; - samples: number; - selfSamples: number; - totalMs: number; - selfMs: number; - children: ProfileNode[]; -}; -export type ProfileFunction = Omit; -export type ParsedProfile = { - root: ProfileNode; - functions: ProfileFunction[]; - sampleCount: number; - durationMs?: number; - threads: { id: string; name: string }[]; - threadId?: string; -}; -function node( - id: string, - name: string, - location = "", - inApp = false, -): ProfileNode { - return { - id, - name, - location, - inApp, - samples: 0, - selfSamples: 0, - totalMs: 0, - selfMs: 0, - children: [], - }; -} -function sampleTime(sample: Record): number | undefined { - const ns = sample.elapsed_since_start_ns; - if (typeof ns === "string" && ns.trim() && Number.isFinite(Number(ns))) - return Number(ns) / 1e6; - if (finite(ns) !== undefined) return Number(ns) / 1e6; - const seconds = finite(sample.timestamp); - return seconds === undefined ? undefined : seconds * 1000; -} -function sampleThread(value: unknown): string { - const number = finite(value); - return text(value) ?? (number === undefined ? "main" : String(number)); -} -/** SDK stacks are leaf-first; sample intervals are calculated independently per thread. */ +export type { + ProfileNode, + ProfileFunction, + ParsedProfile, +} from "@sentry/telemetry"; + export function parseProfile( item: LocalFeedItem, selectedThread?: string, ): ParsedProfile { - const payload = record(item.payload), - profile = Object.keys(record(payload.profile)).length - ? record(payload.profile) - : payload; - const samples = (Array.isArray(profile.samples) ? profile.samples : []).map( - record, - ); - const frames = Array.isArray(profile.frames) ? profile.frames : []; - const stacks = Array.isArray(profile.stacks) ? profile.stacks : []; - const metadata = record(profile.thread_metadata); - const threads = [ - ...new Set(samples.map((sample) => sampleThread(sample.thread_id))), - ].map((id) => ({ id, name: text(record(metadata[id]).name) ?? id })); - const threadId = threads.some((thread) => thread.id === selectedThread) - ? selectedThread - : threads[0]?.id; - const selected = samples - .filter((sample) => sampleThread(sample.thread_id) === threadId) - .sort((a, b) => (sampleTime(a) ?? Infinity) - (sampleTime(b) ?? Infinity)); - const root = node("root", "All samples"), - functions = new Map(); - const children = new Map>(); - let hasTiming = false; - for (let index = 0; index < selected.length; index++) { - const sample = selected[index]!, - stackId = finite(sample.stack_id), - rawStack: unknown = stackId === undefined ? undefined : stacks[stackId]; - const stack = Array.isArray(rawStack) - ? rawStack - .filter( - (id): id is number => - typeof id === "number" && Number.isInteger(id) && id >= 0, - ) - .toReversed() - : []; - const time = sampleTime(sample), - next = selected[index + 1] ? sampleTime(selected[index + 1]!) : undefined; - const weight = - time !== undefined && next !== undefined ? Math.max(0, next - time) : 0; - if (time !== undefined && next !== undefined) hasTiming = true; - root.samples++; - root.totalMs += weight; - let parent = root; - const seenFunctions = new Set(); - // Bound pathological stack depth to protect the UI; raw data remains inspectable. - for (const frameId of stack.slice(0, 256)) { - if (!frames[frameId] || typeof frames[frameId] !== "object") continue; - const frame = record(frames[frameId]); - const name = - text(frame.function) ?? - text(frame.module) ?? - text(frame.instruction_addr) ?? - "Anonymous"; - const location = [ - text(frame.abs_path) ?? text(frame.filename), - finite(frame.lineno), - ] - .filter((value) => value !== undefined) - .join(":"); - const childMap = children.get(parent) ?? new Map(); - children.set(parent, childMap); - let child = childMap.get(frameId); - if (!child) { - child = node( - `${parent.id}/${frameId}`, - name, - location, - frame.in_app === true, - ); - childMap.set(frameId, child); - parent.children.push(child); - } - child.samples++; - child.totalMs += weight; - parent = child; - const key = `${name}:${location}`; - const fn = functions.get(key) ?? { - ...node(key, name, location, frame.in_app === true), - }; - if (!seenFunctions.has(key)) { - fn.samples++; - fn.totalMs += weight; - seenFunctions.add(key); - } - functions.set(key, fn); - } - parent.selfSamples++; - parent.selfMs += weight; - if (parent !== root) { - const fn = functions.get(`${parent.name}:${parent.location}`); - if (fn) { - fn.selfSamples++; - fn.selfMs += weight; - } - } - } - return { - root, - functions: [...functions.values()].sort( - (a, b) => b.totalMs - a.totalMs || b.samples - a.samples, - ), - sampleCount: selected.length, - durationMs: hasTiming ? root.totalMs : undefined, - threads, - threadId, - }; + return buildProfileModel(item.payload, selectedThread); } + type ProfileBounds = { start: number; end: number }; // Captured items are immutable. Reuse bounds across trace rows and inspectors, // and let retention release the cache together with its captured item. diff --git a/apps/local/src/lib/telemetry-regressions.test.ts b/apps/local/src/lib/telemetry-regressions.test.ts new file mode 100644 index 000000000..c38889c2c --- /dev/null +++ b/apps/local/src/lib/telemetry-regressions.test.ts @@ -0,0 +1,76 @@ +import { expect, test } from "vitest"; +import { decodeEnvelope } from "./spotlight.ts"; +import { createLocalTelemetryStore } from "./telemetry-store.ts"; +import { spanInsights } from "./performance.ts"; + +async function capture(items: unknown[]) { + const store = createLocalTelemetryStore(); + const text = JSON.stringify([{}, items]); + store.recordEnvelope( + { id: "wire", type: "envelope", text, payload: text }, + decodeEnvelope(text, "wire"), + ); + await store.drain(); + return store; +} + +test("production log-batch wire inputs remain discoverable in the trace workspace", async () => { + const store = await capture([ + [ + { type: "log" }, + { + items: [ + { + trace_id: "log-trace", + span_id: "log-span", + timestamp: 1700000000, + body: "log-only trace", + level: "info", + }, + ], + }, + ], + ]); + expect(store.getSnapshot().traceGroups).toMatchObject([ + { + id: "log-trace", + logCount: 1, + title: "log-only trace", + startTimestamp: 1700000000, + endTimestamp: 1700000000, + spans: [], + }, + ]); + store.clear(); + expect(store.getSnapshot().traceGroups).toEqual([]); +}); + +test("production span-v2 resource sizes use known wire aliases and omit unknown sizes", async () => { + const attributes = [ + { "http.response.body.size": { type: "integer", value: 200 } }, + { "http.decoded_response_content_length": { type: "integer", value: 400 } }, + {}, + ]; + const store = await capture([ + [ + { type: "span" }, + { + version: 2, + items: attributes.map((data, index) => ({ + trace_id: "resource-trace", + span_id: String(index), + name: "asset.js", + start_timestamp: 1, + end_timestamp: 2, + attributes: { + "sentry.op": { type: "string", value: "resource.script" }, + ...data, + }, + })), + }, + ], + ]); + expect( + spanInsights(store.getSnapshot().traceGroups, "resources"), + ).toMatchObject([{ count: 3, averageBytes: 300, averageMs: 1000 }]); +}); diff --git a/apps/local/src/lib/telemetry-store.test.ts b/apps/local/src/lib/telemetry-store.test.ts index fcb36d73c..79834ea6f 100644 --- a/apps/local/src/lib/telemetry-store.test.ts +++ b/apps/local/src/lib/telemetry-store.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import type { LocalFeedItem } from "./spotlight.ts"; +import { decodeEnvelope, type LocalFeedItem } from "./spotlight.ts"; import { createLocalTelemetryStore } from "./telemetry-store.ts"; function item(overrides: Partial): LocalFeedItem { @@ -106,3 +106,111 @@ describe("LocalTelemetryStore", () => { expect(store.getSnapshot().envelopes).toEqual([envelope]); }); }); + +test("shared processing assembles streamed traces and keeps the original capture", async () => { + const store = createLocalTelemetryStore({ limit: 2 }); + const capture = (id: string, payload: unknown) => { + const text = JSON.stringify([ + {}, + [[{ type: "span" }, { version: 2, items: [payload] }]], + ]); + const envelope = item({ id, type: "envelope", text, payload: text }); + store.recordEnvelope(envelope, decodeEnvelope(text, id)); + return envelope; + }; + const child = capture("child", { + trace_id: "trace", + span_id: "query", + parent_span_id: "root", + name: "SELECT 1", + start_timestamp: 1.1, + end_timestamp: 1.2, + attributes: { "sentry.op": { type: "string", value: "db" } }, + }); + capture("parent", { + trace_id: "trace", + span_id: "root", + name: "GET /orders", + is_segment: true, + start_timestamp: 1, + end_timestamp: 2, + }); + await store.drain(); + expect(store.getSnapshot().traceGroups[0]).toMatchObject({ + title: "GET /orders", + durationMs: 1000, + roots: [{ id: "root", children: [{ id: "query", durationMs: 100 }] }], + }); + expect(store.getSnapshot().envelopes[0]).toBe(child); + expect( + store + .getSnapshot() + .traceGroups[0]?.spans.find((span) => span.id === "query")?.payload, + ).toEqual(JSON.parse(child.text)[1][0][1].items[0]); + + // A retained parent must not keep spans from an evicted child envelope. + const raw = item({ + id: "opaque", + type: "envelope", + text: '[{},[[{"type":"attachment","filename":"raw.txt"},"unmodified"]]]', + }); + store.recordEnvelope(raw, decodeEnvelope(raw.text, raw.id)); + await store.drain(); + expect( + store.getSnapshot().traceGroups[0]?.spans.map((span) => span.id), + ).toEqual(["root"]); + expect(store.getSnapshot().attachments[0]?.payload).toBe("unmodified"); +}); + +test("clearing invalidates queued processing from the previous capture", async () => { + const store = createLocalTelemetryStore(); + const text = JSON.stringify([ + {}, + [ + [ + { type: "transaction" }, + { + event_id: "old", + type: "transaction", + transaction: "old capture", + start_timestamp: 1, + timestamp: 2, + contexts: { trace: { trace_id: "old", span_id: "root" } }, + }, + ], + ], + ]); + store.recordEnvelope( + item({ id: "old", type: "envelope", text }), + decodeEnvelope(text, "old"), + ); + store.clear(); + await store.drain(); + await Promise.resolve(); + expect(store.getSnapshot().traceGroups).toEqual([]); + expect(store.getSnapshot().items).toEqual([]); +}); + +test("uncorrelated transactions stay discoverable without invented trace context", async () => { + const store = createLocalTelemetryStore(); + const text = JSON.stringify([ + {}, + [ + [ + { type: "transaction" }, + { transaction: "GET /standalone", timestamp: 2 }, + ], + ], + ]); + const raw = item({ id: "standalone", type: "envelope", text }); + store.recordEnvelope(raw, decodeEnvelope(text, raw.id)); + await store.drain(); + expect(store.getSnapshot().traceGroups).toMatchObject([ + { id: "standalone:0", title: "GET /standalone", spans: [], itemCount: 1 }, + ]); + expect(store.getSnapshot().envelopes[0]?.text).toBe(text); + expect(store.getSnapshot().items[0]?.payload).toEqual({ + transaction: "GET /standalone", + timestamp: 2, + }); +}); diff --git a/apps/local/src/lib/telemetry-store.ts b/apps/local/src/lib/telemetry-store.ts index 0038f12ef..81641af97 100644 --- a/apps/local/src/lib/telemetry-store.ts +++ b/apps/local/src/lib/telemetry-store.ts @@ -1,3 +1,5 @@ +import { createTelemetryStore } from "@sentry/telemetry"; +import { adaptTraceGroups, type TraceGroup } from "./trace-model.ts"; import { aiItems } from "./inspection.ts"; import { DEFAULT_MAX_ENVELOPES, @@ -8,6 +10,7 @@ import { isErrorEvent } from "./workspace.ts"; export type LocalTelemetrySnapshot = { items: LocalFeedItem[]; + traceGroups: TraceGroup[]; errors: LocalFeedItem[]; traces: LocalFeedItem[]; logs: LocalFeedItem[]; @@ -26,12 +29,14 @@ export type LocalTelemetryStore = { ) => void; setLimit: (limit: number) => void; clear: () => void; + drain: () => Promise; getSnapshot: () => LocalTelemetrySnapshot; subscribe: (listener: () => void) => () => void; }; const EMPTY_SNAPSHOT: LocalTelemetrySnapshot = { items: [], + traceGroups: [], errors: [], traces: [], logs: [], @@ -46,9 +51,11 @@ const EMPTY_SNAPSHOT: LocalTelemetrySnapshot = { function buildSnapshot( items: LocalFeedItem[], envelopes: LocalFeedItem[], + traceGroups: TraceGroup[] = [], ): LocalTelemetrySnapshot { return { items, + traceGroups, errors: items.filter(isErrorEvent), traces: items.filter( (item) => item.type === "transaction" || item.type === "span", @@ -81,9 +88,88 @@ export function createLocalTelemetryStore({ let snapshot = EMPTY_SNAPSHOT; const envelopeItems = new Map(); const listeners = new Set<() => void>(); + // The raw capture is synchronous. Shared processing is serialized so bursts + // cannot overflow the SDK queue, and reset invalidates work from older captures. + let processor = createProcessor(); + let processing = Promise.resolve(); + let generation = 0; + let processorLimit = limit; + let needsRebuild = false; + let scheduled = false; + let dirty = false; + const processedRawIds = new Set(); + function createProcessor() { + return createTelemetryStore({ + retention: { maxEnvelopes: limit, maxBytes: Number.MAX_SAFE_INTEGER }, + maxEnvelopeBytes: Number.MAX_SAFE_INTEGER, + }); + } + function parsedEnvelope(envelope: LocalFeedItem): unknown { + try { + return JSON.parse(envelope.text); + } catch { + return undefined; + } + } + function updateProcessed() { + dirty = true; + if (scheduled) return; + scheduled = true; + const epoch = generation; + processing = Promise.resolve().then(async () => { + try { + // Coalesce a burst into the latest retained capture instead of retaining + // an unbounded promise chain or an envelope-list copy per delivery. + while (dirty && epoch === generation) { + dirty = false; + const retained = snapshot.envelopes; + const retainedIds = new Set(retained.map((raw) => raw.id)); + if ( + needsRebuild || + [...processedRawIds].some((id) => !retainedIds.has(id)) + ) { + processor.reset(); + processedRawIds.clear(); + needsRebuild = false; + } + if (processorLimit !== limit) { + const nextLimit = limit; + await processor.setRetention({ maxEnvelopes: nextLimit }); + if (epoch !== generation) return; + processorLimit = nextLimit; + } + for (const raw of retained) { + if (epoch !== generation) return; + if (processedRawIds.has(raw.id)) continue; + await processor.ingestEnvelope(parsedEnvelope(raw), { + deliveryId: raw.id, + }); + if (epoch !== generation) return; + processedRawIds.add(raw.id); + } + if (epoch !== generation) return; + publish( + snapshot.items, + snapshot.envelopes, + adaptTraceGroups(snapshot.items, processor.getTraces()), + ); + } + } catch { + // A malformed capture must not poison later deliveries. Keep its raw data + // inspectable and rebuild derived state on the next successful operation. + if (epoch === generation) needsRebuild = true; + } finally { + if (epoch === generation) scheduled = false; + } + }); + } - function publish(items: LocalFeedItem[], envelopes = snapshot.envelopes) { - snapshot = buildSnapshot(items, envelopes); + function publish( + items: LocalFeedItem[], + envelopes = snapshot.envelopes, + traceGroups = snapshot.traceGroups, + ) { + snapshot = buildSnapshot(items, envelopes, traceGroups); for (const listener of listeners) { listener(); } @@ -107,12 +193,25 @@ export function createLocalTelemetryStore({ publish( envelopes.flatMap((item) => [...(envelopeItems.get(item.id) ?? [])]), envelopes, + adaptTraceGroups( + envelopes.flatMap((item) => [...(envelopeItems.get(item.id) ?? [])]), + processor.getTraces(), + ), ); + updateProcessed(); }, + drain: () => processing, clear() { + generation++; + processor.reset(); + processing = Promise.resolve(); + dirty = false; + scheduled = false; + needsRebuild = false; + processedRawIds.clear(); envelopeItems.clear(); if (snapshot.items.length > 0 || snapshot.envelopes.length > 0) { - publish([], []); + publish([], [], []); } }, getSnapshot: () => snapshot, diff --git a/apps/local/src/lib/trace-model.test.ts b/apps/local/src/lib/trace-model.test.ts index 32c6c10bd..185afd32b 100644 --- a/apps/local/src/lib/trace-model.test.ts +++ b/apps/local/src/lib/trace-model.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "vitest"; import { decodeEnvelope, type LocalFeedItem } from "./spotlight.js"; -import { buildTraceGroups, traceIdForItem } from "./trace-model.js"; +import { traceIdForItem } from "./trace-model.js"; +import { buildTraceGroups } from "../test-fixtures/trace-groups.ts"; function item(overrides: Partial): LocalFeedItem { return { @@ -16,7 +17,7 @@ function item(overrides: Partial): LocalFeedItem { describe("buildTraceGroups", () => { test.each([undefined, "remote-parent"])( "promotes a late streamed segment title and reparents earlier children (parent: %s)", - (parentId) => { + async (parentId) => { const child = { trace_id: "trace-stream", span_id: "query", @@ -32,7 +33,7 @@ describe("buildTraceGroups", () => { id, ); const earlyItems = decode([child], "child"); - expect(buildTraceGroups(earlyItems)[0]?.roots).toMatchObject([ + expect((await buildTraceGroups(earlyItems))[0]?.roots).toMatchObject([ { id: "query" }, ]); const items = [ @@ -56,7 +57,7 @@ describe("buildTraceGroups", () => { ), ...decode([child], "redelivery"), ]; - const [trace] = buildTraceGroups(items); + const [trace] = await buildTraceGroups(items); expect(trace).toMatchObject({ title: "GET /orders", startTimestamp: 1700000000, @@ -73,7 +74,7 @@ describe("buildTraceGroups", () => { test.each([true, false])( "combines legacy transactions and streamed spans without duplicate spans (transaction first: %s)", - (transactionFirst) => { + async (transactionFirst) => { const transaction = decodeEnvelope( JSON.stringify([ {}, @@ -141,7 +142,7 @@ describe("buildTraceGroups", () => { ]), "streamed", ); - const traces = buildTraceGroups( + const traces = await buildTraceGroups( transactionFirst ? [...transaction, ...streamed] : [...streamed, ...transaction], @@ -158,7 +159,7 @@ describe("buildTraceGroups", () => { }, ); - test("reads trace timing in linear work for a long single trace", () => { + test("reads trace timing in linear work for a long single trace", async () => { let timestampReads = 0; const entries: LocalFeedItem[] = Array.from( { length: 1000 }, @@ -174,7 +175,7 @@ describe("buildTraceGroups", () => { }, }), ); - const [trace] = buildTraceGroups(entries); + const [trace] = await buildTraceGroups(entries); expect(trace).toMatchObject({ itemCount: 1000, logCount: 1000, @@ -185,14 +186,14 @@ describe("buildTraceGroups", () => { expect(timestampReads).toBeLessThanOrEqual(entries.length * 2); }); - test("handles timing for a valid transaction with many child spans without argument overflow", () => { + test("handles timing for a valid transaction with many child spans without argument overflow", async () => { const spans = Array.from({ length: 70000 }, (_, index) => ({ span_id: String(index), parent_span_id: "root", start_timestamp: index, timestamp: index + 1, })); - const [trace] = buildTraceGroups([ + const [trace] = await buildTraceGroups([ item({ metadata: { title: "large transaction", traceId: "large-trace" }, payload: { @@ -214,7 +215,7 @@ describe("buildTraceGroups", () => { expect(trace?.roots[0]?.children).toHaveLength(70000); }); - test("uses a transaction ID as the trace fallback used by routing", () => { + test("uses a transaction ID as the trace fallback used by routing", async () => { expect( traceIdForItem( item({ @@ -225,8 +226,8 @@ describe("buildTraceGroups", () => { ).toBe("transaction-without-trace"); }); - test("keeps a standalone transaction discoverable when it has no trace context", () => { - const traces = buildTraceGroups([ + test("keeps a standalone transaction discoverable when it has no trace context", async () => { + const traces = await buildTraceGroups([ { id: "transaction-without-trace", type: "transaction", @@ -242,8 +243,8 @@ describe("buildTraceGroups", () => { expect(traces[0]?.title).toBe("GET /standalone"); }); - test("builds a timestamped span tree and correlates trace errors and logs", () => { - const traces = buildTraceGroups([ + test("builds a timestamped span tree and correlates trace errors and logs", async () => { + const traces = await buildTraceGroups([ item({ id: "transaction", timestamp: 1_700_000_000.012, @@ -328,8 +329,8 @@ describe("buildTraceGroups", () => { ]); }); - test("keeps a span visible when its declared parent is absent", () => { - const traces = buildTraceGroups([ + test("keeps a span visible when its declared parent is absent", async () => { + const traces = await buildTraceGroups([ item({ metadata: { title: "orphaned work", @@ -356,8 +357,8 @@ describe("buildTraceGroups", () => { ]); }); - test("nests a standalone span using its top-level parent span id", () => { - const traces = buildTraceGroups([ + test("nests a standalone span using its top-level parent span id", async () => { + const traces = await buildTraceGroups([ item({ id: "root", metadata: { @@ -409,7 +410,7 @@ describe("buildTraceGroups", () => { }); }); -test("child spans retain their own raw payload independently of the transaction", () => { +test("child spans retain their own raw payload independently of the transaction", async () => { const payload = { span_id: "query", parent_span_id: "root", @@ -418,7 +419,7 @@ test("child spans retain their own raw payload independently of the transaction" start_timestamp: 1.1, timestamp: 1.2, }; - const [trace] = buildTraceGroups( + const [trace] = await buildTraceGroups( decodeEnvelope( JSON.stringify([ {}, diff --git a/apps/local/src/lib/trace-model.ts b/apps/local/src/lib/trace-model.ts index f38be8afe..c1e646167 100644 --- a/apps/local/src/lib/trace-model.ts +++ b/apps/local/src/lib/trace-model.ts @@ -1,9 +1,11 @@ +import { type ProcessedTrace, type ProcessedSpan } from "@sentry/telemetry"; import { getEventMetadata, type EventMetadata, type LocalFeedItem, -} from "./spotlight.js"; -import { isErrorEvent } from "./workspace.js"; +} from "./spotlight.ts"; +import { record, text } from "./inspection.ts"; +import { isErrorEvent } from "./workspace.ts"; export type TraceSpan = { id: string; @@ -21,6 +23,7 @@ export type TraceSpan = { export type TraceGroup = { id: string; + processed?: ProcessedTrace; title: string; items: LocalFeedItem[]; roots: TraceSpan[]; @@ -33,126 +36,6 @@ export type TraceGroup = { durationMs?: number; }; -type MutableTraceGroup = Omit & { - spanMap: Map; - titlePriority: number; -}; - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function getString(value: unknown): string | undefined { - return typeof value === "string" && value ? value : undefined; -} - -function getTimestamp(value: unknown): number | undefined { - if (typeof value === "number" && Number.isFinite(value)) { - return value; - } - if (typeof value === "string" && value.trim()) { - const parsed = Number(value); - return Number.isFinite(parsed) ? parsed : undefined; - } - return undefined; -} - -function getTraceContext(payload: unknown): Record { - if (!isRecord(payload)) { - return {}; - } - const contexts = isRecord(payload.contexts) ? payload.contexts : {}; - return isRecord(contexts.trace) ? contexts.trace : {}; -} - -function getPayloadSpan( - payload: unknown, - item: LocalFeedItem, -): TraceSpan | undefined { - const event = isRecord(payload) ? payload : {}; - const trace = getTraceContext(payload); - const id = getString(trace.span_id) ?? item.metadata?.spanId; - if (!id) { - return undefined; - } - const startTimestamp = getTimestamp(event.start_timestamp); - const endTimestamp = - getTimestamp(event.timestamp) ?? - getTimestamp(event.end_timestamp) ?? - getTimestamp(item.timestamp); - const operation = getString(trace.op) ?? item.metadata?.operation; - const description = - getString(event.transaction) ?? - item.metadata?.route ?? - item.metadata?.title ?? - operation ?? - item.type; - - return { - id, - parentId: - getString(trace.parent_span_id) ?? getString(event.parent_span_id), - operation, - description, - startTimestamp, - endTimestamp, - durationMs: getDurationMs(startTimestamp, endTimestamp), - item, - payload, - metadata: item.metadata ?? getEventMetadata(payload, item.type), - children: [], - }; -} - -function getChildSpans(payload: unknown, item: LocalFeedItem): TraceSpan[] { - const event = isRecord(payload) ? payload : {}; - if (!Array.isArray(event.spans)) { - return []; - } - - return event.spans.flatMap((value) => { - if (!isRecord(value)) { - return []; - } - const id = getString(value.span_id); - if (!id) { - return []; - } - const startTimestamp = getTimestamp(value.start_timestamp); - const endTimestamp = getTimestamp(value.timestamp); - const operation = getString(value.op); - return [ - { - id, - parentId: getString(value.parent_span_id), - operation, - description: getString(value.description) ?? operation ?? id, - startTimestamp, - endTimestamp, - durationMs: getDurationMs(startTimestamp, endTimestamp), - item, - payload: value, - metadata: getEventMetadata(value, "span"), - children: [], - }, - ]; - }); -} - -function getDurationMs( - startTimestamp: number | undefined, - endTimestamp: number | undefined, -): number | undefined { - if ( - startTimestamp === undefined || - endTimestamp === undefined || - endTimestamp < startTimestamp - ) { - return undefined; - } - return Math.round((endTimestamp - startTimestamp) * 100_000) / 100; -} - export function traceIdForItem(item: LocalFeedItem): string | undefined { return ( item.metadata?.traceId ?? @@ -160,126 +43,152 @@ export function traceIdForItem(item: LocalFeedItem): string | undefined { ); } -function sortByTimestamp( - items: T[], -): T[] { - return items.sort( - (left, right) => - (left.startTimestamp ?? Infinity) - (right.startTimestamp ?? Infinity), - ); -} - -function updateTraceTiming(trace: MutableTraceGroup): void { - let start = Infinity; - let end = -Infinity; - function includeTimestamp(timestamp: number | undefined): void { - if (timestamp === undefined) return; - start = Math.min(start, timestamp); - end = Math.max(end, timestamp); - } - for (const item of trace.items) - includeTimestamp(getTimestamp(item.timestamp)); - for (const span of trace.spanMap.values()) { - includeTimestamp(span.startTimestamp); - includeTimestamp(span.endTimestamp); - } - if (start === Infinity) return; - trace.startTimestamp = start; - trace.endTimestamp = end; - trace.durationMs = getDurationMs(start, end); -} - -/** Build a small, read-only trace model once for every view in Local. */ -export function buildTraceGroups( +/** Adapt shared trace assembly to Local's raw-item inspection and display model. + * Local's presentation timestamps are wire seconds; the SDK uses milliseconds. + */ +export function adaptTraceGroups( items: readonly LocalFeedItem[], + processed: readonly ProcessedTrace[], ): TraceGroup[] { - const traces = new Map(); - + const itemsByTrace = new Map(); + const rawSpans = new Map(); for (const item of items) { - const traceId = traceIdForItem(item); - if (!traceId) { - continue; - } - - const trace: MutableTraceGroup = traces.get(traceId) ?? { - id: traceId, - title: item.metadata?.title ?? item.type, - titlePriority: 0, - items: [], - logCount: 0, - errorCount: 0, - spanMap: new Map(), - }; - if (!traces.has(traceId)) { - traces.set(traceId, trace); - } - - trace.items.push(item); - if (item.type === "log") { - trace.logCount += 1; - } - if (isErrorEvent(item)) { - trace.errorCount += 1; - } - - if (item.type === "transaction" || item.type === "span") { - const rootSpan = getPayloadSpan(item.payload, item); - if (rootSpan) { - trace.spanMap.set(rootSpan.id, rootSpan); - const isSegment = - isRecord(item.payload) && item.payload.is_segment === true; - if (item.type === "transaction" || isSegment) { - // A local segment can have a remote parent absent from this receiver. - const priority = rootSpan.parentId ? 1 : 2; - if (priority > trace.titlePriority) { - trace.title = item.metadata?.title ?? rootSpan.description; - trace.titlePriority = priority; - } - } - } - } - for (const childSpan of getChildSpans(item.payload, item)) { - trace.spanMap.set(childSpan.id, childSpan); + const id = traceIdForItem(item); + if (!id) continue; + const entries = itemsByTrace.get(id) ?? []; + entries.push(item); + itemsByTrace.set(id, entries); + if ( + (item.type === "transaction" || item.type === "span") && + item.metadata?.spanId + ) + rawSpans.set(`${id}:${item.metadata.spanId}`, { + item, + payload: item.payload, + }); + const payload = record(item.payload); + for (const span of Array.isArray(payload.spans) ? payload.spans : []) { + const spanId = text(record(span).span_id); + if (spanId) rawSpans.set(`${id}:${spanId}`, { item, payload: span }); } } - - return [...traces.values()] - .map((trace) => { - updateTraceTiming(trace); - const spans = sortByTimestamp([...trace.spanMap.values()]); - const roots: TraceSpan[] = []; - for (const span of spans) { - span.children = []; + const assembled = processed + .flatMap((trace) => { + const entries = itemsByTrace.get(trace.trace_id); + if (!entries?.length) return []; + const mapped = new Map(); + function adapt(span: ProcessedSpan): TraceSpan { + const previous = mapped.get(span.span_id); + if (previous) return previous; + const raw = rawSpans.get(`${trace.trace_id}:${span.span_id}`); + const item = raw?.item ?? entries![0]!; + const payload = raw?.payload ?? span; + const duration = span.timestamp - span.start_timestamp; + const result: TraceSpan = { + id: span.span_id, + parentId: span.parent_span_id ?? undefined, + operation: span.op ?? undefined, + description: span.description ?? span.op ?? span.span_id, + startTimestamp: span.start_timestamp / 1000, + endTimestamp: span.timestamp / 1000, + durationMs: + duration >= 0 && Number.isFinite(duration) + ? Math.round(duration * 100) / 100 + : undefined, + item, + payload, + metadata: getEventMetadata(payload, "span"), + children: [], + }; + mapped.set(span.span_id, result); + return result; } - for (const span of spans) { - const parent = span.parentId - ? trace.spanMap.get(span.parentId) - : undefined; - if (parent && parent !== span) { - parent.children.push(span); + // Convert the SDK's existing tree iteratively; very deep captures must not + // use the JavaScript call stack merely to create presentation objects. + const sharedSpans = [...trace.spans.values()]; + for (const span of sharedSpans) if (span.op !== "orphan") adapt(span); + for (const span of sharedSpans) { + const local = mapped.get(span.span_id); + if (local) + local.children = (span.children ?? []).flatMap((child) => { + const converted = mapped.get(child.span_id); + return converted ? [converted] : []; + }); + } + const roots: TraceSpan[] = []; + const pending = [...trace.spanTree].reverse(); + while (pending.length) { + const span = pending.pop()!; + if (span.op === "orphan") { + for ( + let index = (span.children?.length ?? 0) - 1; + index >= 0; + index-- + ) + pending.push(span.children![index]!); } else { - roots.push(span); + roots.push(adapt(span)); } } - for (const span of spans) { - sortByTimestamp(span.children); - } - + return [ + { + id: trace.trace_id, + processed: trace, + title: + trace.rootTransaction?.transaction ?? + trace.transactions[0]?.transaction ?? + entries[0]!.metadata?.title ?? + trace.rootTransactionName, + items: entries, + roots, + spans: [...mapped.values()].sort( + (a, b) => + (a.startTimestamp ?? Infinity) - (b.startTimestamp ?? Infinity), + ), + logCount: entries.filter((item) => item.type === "log").length, + errorCount: entries.filter(isErrorEvent).length, + itemCount: entries.length, + startTimestamp: trace.start_timestamp / 1000, + endTimestamp: trace.timestamp / 1000, + durationMs: + Math.round((trace.timestamp - trace.start_timestamp) * 100) / 100, + }, + ]; + }) + .sort((a, b) => (b.endTimestamp ?? 0) - (a.endTimestamp ?? 0)); + // Local also presents standalone transactions that carry no trace context. + // These are navigation groups for captured records, without invented SDK spans. + const assembledIds = new Set(assembled.map((trace) => trace.id)); + const uncorrelated: TraceGroup[] = items + .filter( + (item) => + item.type === "transaction" && + !item.metadata?.traceId && + !assembledIds.has(item.id), + ) + .map((item) => { + const payload = record(item.payload); + const start = + typeof payload.start_timestamp === "number" + ? payload.start_timestamp + : undefined; + const end = + typeof payload.timestamp === "number" ? payload.timestamp : undefined; return { - id: trace.id, - title: trace.title, - items: trace.items, - roots: sortByTimestamp(roots), - spans, - logCount: trace.logCount, - errorCount: trace.errorCount, - itemCount: trace.items.length, - startTimestamp: trace.startTimestamp, - endTimestamp: trace.endTimestamp, - durationMs: trace.durationMs, + id: item.id, + title: item.metadata?.title ?? "transaction", + items: [item], + roots: [], + spans: [], + logCount: 0, + errorCount: 0, + itemCount: 1, + startTimestamp: start, + endTimestamp: end, + durationMs: item.metadata?.durationMs, }; - }) - .sort( - (left, right) => (right.endTimestamp ?? 0) - (left.endTimestamp ?? 0), - ); + }); + return [...assembled, ...uncorrelated].sort( + (a, b) => (b.endTimestamp ?? 0) - (a.endTimestamp ?? 0), + ); } diff --git a/apps/local/src/test-fixtures/trace-groups.ts b/apps/local/src/test-fixtures/trace-groups.ts new file mode 100644 index 000000000..c257055a1 --- /dev/null +++ b/apps/local/src/test-fixtures/trace-groups.ts @@ -0,0 +1,112 @@ +import { createTelemetryStore } from "@sentry/telemetry"; +import { + adaptTraceGroups, + traceIdForItem, + type TraceGroup, +} from "../lib/trace-model.ts"; +import type { LocalFeedItem } from "../lib/spotlight.ts"; +import { record } from "../lib/inspection.ts"; + +/** Reconstruct a parsed wire envelope for callers that already decoded its items. */ +export function telemetryEnvelope(items: readonly LocalFeedItem[]): unknown { + const fallbackTimestamp = + items + .map((item) => record(item.payload).timestamp ?? item.timestamp) + .find((value) => value !== undefined) ?? 0; + return [ + items[0]?.envelopeHeader ?? {}, + items.map((item) => { + const payload = record(item.payload); + const metadata = item.metadata; + const contexts = record(payload.contexts); + const trace = record(contexts.trace); + const traceId = traceIdForItem(item); + const timestamp = + payload.timestamp ?? + payload.end_timestamp ?? + item.timestamp ?? + fallbackTimestamp; + const enriched = { + ...payload, + event_id: payload.event_id ?? item.id, + timestamp, + ...(traceId + ? { + contexts: { + ...contexts, + trace: { + trace_id: traceId, + span_id: metadata?.spanId ?? item.id, + op: metadata?.operation, + ...trace, + }, + }, + } + : {}), + }; + if (item.type === "transaction") { + Object.assign(enriched, { + type: "transaction", + transaction: payload.transaction ?? metadata?.title, + start_timestamp: payload.start_timestamp ?? timestamp, + }); + } + if (item.type === "span") { + // Decoded span-v2 items have typed attributes. Pass a v2 container to + // the shared processor rather than reimplementing attribute normalization. + const span = { + ...payload, + trace_id: payload.trace_id ?? traceId, + span_id: payload.span_id ?? metadata?.spanId ?? item.id, + op: payload.op ?? metadata?.operation, + description: payload.description ?? metadata?.title, + timestamp, + }; + return [ + item.itemHeader ?? { type: item.type }, + "attributes" in payload + ? { + version: 2, + items: [ + { + ...span, + end_timestamp: payload.end_timestamp ?? timestamp, + }, + ], + } + : span, + ]; + } + if (item.type === "log") { + const log = { + ...payload, + timestamp, + trace_id: payload.trace_id ?? traceId, + level: payload.level ?? metadata?.level ?? "info", + body: payload.body ?? metadata?.title ?? "", + }; + return [ + item.itemHeader ?? { type: item.type }, + { ...enriched, items: [log] }, + ]; + } + return [item.itemHeader ?? { type: item.type }, enriched]; + }), + ]; +} + +/** Convenience for isolated captures; the live workspace uses its session store. */ +export async function buildTraceGroups( + items: readonly LocalFeedItem[], +): Promise { + const store = createTelemetryStore({ + maxEnvelopeBytes: 128 * 1024 * 1024, + retention: { maxBytes: 128 * 1024 * 1024 }, + }); + try { + await store.ingestEnvelope(telemetryEnvelope(items)); + return adaptTraceGroups(items, store.getTraces()); + } finally { + store.dispose(); + } +} diff --git a/docs/README.md b/docs/README.md index c74d1a52c..954f04278 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,8 @@ and package it from that workspace. The contributor index here links to its source and to . MCP documentation lives in the topic directories below; the MCP index explains which guides apply to that product. +Shared telemetry processing for the receiver and browser UI lives in [`@sentry/telemetry`](../packages/telemetry/README.md). + ## Shared Contributor Guides - [Coding guidelines](contributing/coding-guidelines.md) — repository-wide style diff --git a/docs/cli/local-control-plane.md b/docs/cli/local-control-plane.md index 7eab6eb32..24fb831e8 100644 --- a/docs/cli/local-control-plane.md +++ b/docs/cli/local-control-plane.md @@ -77,6 +77,13 @@ Increasing the limit preserves existing envelopes, their IDs, and active streams previously evicted envelopes cannot be recovered. Lowering below the retained count is rejected without changing any data. Clear envelopes first to reduce that limit. Changing capacity does not reset the session's three-hour lifecycle retention. +The receiver decodes and processes envelopes with `@sentry/telemetry/node`, shared +with the browser viewer's telemetry engine. Envelope retention also has a 64 MiB +parsed-data budget. Raw downloads have a separate 64 MiB decoded-body budget; +a raw download returns 404 once its decoded bytes have been evicted. Both budgets +are in memory and clear with the session. Malformed envelopes and invalid compressed +bodies return HTTP 400 without entering retained history. The existing ingest URLs, +SSE event names, resume IDs, and opaque attachment representation are preserved. An explicit limit also configures an existing plain receiver; ordinary attachment preserves its current setting. Older receivers that cannot apply the flag report an error. Retrying session creation with the same client ID preserves its original settings. @@ -240,8 +247,8 @@ the loopback receiver. Other browser origins cannot change retention. Remote HTT Standalone spans and version 2 span containers populate trace waterfalls as they arrive. Child spans remain visible before their parent or segment arrives; the segment supplies the trace name once available. Typed span attributes supply -operation and HTTP metadata. Unsupported containers and malformed spans are -skipped by the trace view while their raw envelopes remain inspectable. Legacy +operation and HTTP metadata. Unsupported containers are retained without trace processing. Envelopes containing +only malformed spans are rejected; valid items in mixed envelopes are retained. Legacy standalone spans and transaction envelopes remain supported. Exception event titles use the current SDK exception type and message, including diff --git a/packages/cli/package.json b/packages/cli/package.json index ba3e9e9d1..5221fd07a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -61,7 +61,7 @@ "build": "pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/build.ts --single", "build:all": "pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/build.ts", "bundle": "pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/bundle.ts", - "typecheck": "pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:docs && pnpm run generate:sdk && tsc --noEmit", + "typecheck": "pnpm --filter @sentry/telemetry run build && pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:docs && pnpm run generate:sdk && tsc --noEmit", "lint": "pnpm -w exec oxlint --report-unused-disable-directives packages/cli && pnpm -w exec oxfmt --check packages/cli", "lint:fix": "pnpm -w exec oxlint --fix packages/cli && pnpm -w exec oxfmt --write packages/cli", "test": "pnpm run test:unit", @@ -76,7 +76,7 @@ "generate:grammar": "node ../../scripts/generate-parsers.mjs .", "generate:completions": "pnpm tsx script/generate-completion-targets.ts", "generate:docs-sections": "pnpm tsx script/generate-docs-sections.ts", - "generate:schema": "pnpm tsx script/generate-api-schema.ts", + "generate:schema": "pnpm --filter @sentry/telemetry run build && pnpm tsx script/generate-api-schema.ts", "generate:command-docs": "pnpm tsx script/generate-command-docs.ts", "eval:skill": "pnpm tsx script/eval-skill.ts", "bench": "pnpm tsx script/bench.ts", @@ -108,7 +108,7 @@ "@sentry/sqlish": "^1.0.1", "@sentry/symbolic": "13.7.0", "@sentry/toolkit-core": "workspace:*", - "@spotlightjs/spotlight": "^4.11.7", + "@sentry/telemetry": "workspace:*", "@stricli/auto-complete": "^1.2.8", "@stricli/core": "1.2.8", "@types/http-cache-semantics": "^4.2.0", diff --git a/packages/cli/src/commands/local/control-plane.ts b/packages/cli/src/commands/local/control-plane.ts index b4fcd2f73..334c6ad49 100644 --- a/packages/cli/src/commands/local/control-plane.ts +++ b/packages/cli/src/commands/local/control-plane.ts @@ -6,8 +6,11 @@ */ import { randomBytes } from "node:crypto"; +import { + localEnvelopeContentType, + localIngestStatus, +} from "./telemetry-buffer.js"; import { Buffer } from "node:buffer"; -import { pushToSpotlightBuffer } from "@spotlightjs/spotlight/sdk"; import { type Context, Hono } from "hono"; import { cors } from "hono/cors"; import { HTTPException } from "hono/http-exception"; @@ -407,13 +410,7 @@ export function createLocalControlPlane({ return c.body(null, 413); } const body = Buffer.from(arrayBuffer); - const rawEncoding = c.req.header("content-encoding"); - const encoding = - rawEncoding === "gzip" || - rawEncoding === "deflate" || - rawEncoding === "br" - ? rawEncoding - : undefined; + const encoding = c.req.header("content-encoding"); const browserBeacon = c.req.query("sentry_client")?.startsWith("sentry.javascript.browser") && c.req.header("origin"); @@ -423,15 +420,17 @@ export function createLocalControlPlane({ { error: "Session is closed; create a new session to capture events." }, 410, ); - pushToSpotlightBuffer({ - spotlightBuffer: sessionStore.getBuffer(reference), + const receipt = await sessionStore.getBuffer(reference).ingestPayload({ body, contentType: browserBeacon ? SENTRY_CONTENT_TYPE - : (c.req.header("content-type") ?? ""), + : localEnvelopeContentType(c.req.header("content-type")), encoding, userAgent: c.req.header("user-agent"), }); + const status = localIngestStatus(receipt); + if (status === 503) c.header("Retry-After", "1"); + if (status !== 204) return c.body(null, status); sessionStore.recordActivity(reference); return c.body(null, 204); }; @@ -491,9 +490,17 @@ export function createLocalControlPlane({ }); const unsubscribeSession = sessionStore.subscribe(reference, (event) => { if (event.state === "retained" && event.action !== "reset") { - buffer.stream(readerId); - buffer.unsubscribe(readerId); - void sendSession(event).finally(finish); + void buffer + .drain() + .then(async () => { + buffer.stream(readerId); + buffer.unsubscribe(readerId); + await sendSession(event); + }) + .catch((error: unknown) => + logger.debug("Local session close failed", error), + ) + .finally(finish); } else { void sendSession(event); } @@ -502,6 +509,7 @@ export function createLocalControlPlane({ try { const { state, expiresAt } = sessionStore.resolve(reference); if (state === "retained") { + await buffer.drain(); buffer.stream(readerId); await sendSession({ state, expiresAt }); } else { @@ -525,7 +533,7 @@ export function createLocalControlPlane({ : undefined; try { validateMaxEnvelopes(value); - sessionStore.setMaxEnvelopes(c.req.param("id") ?? "", value); + await sessionStore.setMaxEnvelopes(c.req.param("id") ?? "", value); return c.json( sessionStore.getRetention(c.req.param("id") ?? "").getSettings(), ); @@ -654,15 +662,15 @@ export function createLocalControlPlane({ if (!sessionExists(c.req.param("id"))) { return c.body(null, 404); } - const container = sessionStore + const payload = sessionStore .getBuffer(c.req.param("id")) - .read({ envelopeId: c.req.param("envelopeId") })[0]; - if (!container) { + .pipeline.getDecodedPayload(c.req.param("envelopeId")); + if (!payload) { return c.body(null, 404); } return c.json({ - contentType: container.getContentType(), - data: Buffer.from(container.getData()).toString("base64"), + contentType: payload.contentType, + data: payload.body.toString("base64"), id: c.req.param("envelopeId"), }); }); diff --git a/packages/cli/src/commands/local/retention-buffer.ts b/packages/cli/src/commands/local/retention-buffer.ts index 9f4f13fe9..2fc2596fa 100644 --- a/packages/cli/src/commands/local/retention-buffer.ts +++ b/packages/cli/src/commands/local/retention-buffer.ts @@ -1,5 +1,8 @@ -/** Resize through the Spotlight SDK's public API while keeping a stable buffer facade. */ -import { createSpotlightBuffer } from "@spotlightjs/spotlight/sdk"; +/** Receiver retention controls backed by the shared telemetry pipeline. */ +import { + createLocalTelemetryBuffer, + type LocalTelemetryBuffer, +} from "./telemetry-buffer.js"; import { ValidationError } from "../../lib/errors.js"; import { DEFAULT_MAX_ENVELOPES, @@ -7,19 +10,14 @@ import { MAX_ENVELOPES, } from "./retention-limits.js"; -type SpotlightBuffer = ReturnType; -type Subscriber = { - callback: Parameters[0]; - id: string; -}; export type RetentionSettings = { maxEnvelopes: number; retainedEnvelopes: number; }; export type LocalRetentionBuffer = { - buffer: SpotlightBuffer; + buffer: LocalTelemetryBuffer; getSettings: () => RetentionSettings; - resize: (maxEnvelopes: number) => void; + resize: (maxEnvelopes: number) => Promise; subscribeSettings: ( listener: (settings: RetentionSettings) => void, ) => () => void; @@ -65,78 +63,29 @@ export function createLocalRetentionBuffer( maxEnvelopes = DEFAULT_MAX_ENVELOPES, ): LocalRetentionBuffer { validateMaxEnvelopes(maxEnvelopes); - let delegate = createSpotlightBuffer(maxEnvelopes); - const buffer = createSpotlightBuffer(1); - const subscribers = new Map(); - const arrivalTimes = new WeakMap(); + const buffer = createLocalTelemetryBuffer(maxEnvelopes); const listeners = new Set<(settings: RetentionSettings) => void>(); const getSettings = (): RetentionSettings => ({ maxEnvelopes, - retainedEnvelopes: delegate.read({ all: true }).length, + retainedEnvelopes: buffer.read({ all: true }).length, }); - buffer.put = (item) => { - arrivalTimes.set(item, Date.now()); - delegate.put(item); - }; - buffer.read = (filters = { timeWindow: 60 }) => { - if ("timeWindow" in filters && filters.timeWindow !== undefined) { - const { timeWindow, ...rest } = filters; - return delegate - .read({ ...rest, all: true }) - .filter( - (item) => - (arrivalTimes.get(item) ?? 0) > Date.now() - timeWindow * 1000, - ); - } - return delegate.read(filters); - }; - buffer.subscribe = (callback, lastEventId) => { - const id = delegate.subscribe(callback, lastEventId); - subscribers.set(id, { callback, id }); - return id; - }; - buffer.unsubscribe = (id) => { - const subscriber = subscribers.get(id); - if (subscriber) delegate.unsubscribe(subscriber.id); - subscribers.delete(id); - }; - buffer.stream = (id) => { - const subscriber = subscribers.get(id); - if (subscriber) delegate.stream(subscriber.id); - }; - buffer.clear = () => delegate.clear(); - buffer.reset = () => delegate.reset(); return { buffer, getSettings, resize(nextLimit) { validateMaxEnvelopes(nextLimit); - const retained = delegate.read({ all: true }); - if (nextLimit < retained.length) - throw new RetentionLimitError( - `Cannot lower the limit below ${retained.length} retained envelopes. Clear envelopes first.`, - 409, - ); - if (nextLimit === maxEnvelopes) return; - // Flush queued deliveries before moving readers, then advance each new reader - // past the copied history so an active stream neither misses nor repeats it. - for (const subscriber of subscribers.values()) - delegate.stream(subscriber.id); - const next = createSpotlightBuffer(nextLimit); - for (const item of retained.reverse()) next.put(item); - const previous = delegate; - delegate = next; - for (const subscriber of subscribers.values()) { - previous.unsubscribe(subscriber.id); - let replaying = true; - subscriber.id = next.subscribe((item) => { - if (!replaying) subscriber.callback(item); - }); - next.stream(subscriber.id); - replaying = false; - } - maxEnvelopes = nextLimit; - for (const listener of listeners) listener(getSettings()); + return buffer.withWriteBarrier(async () => { + const retained = buffer.read({ all: true }); + if (nextLimit < retained.length) + throw new RetentionLimitError( + `Cannot lower the limit below ${retained.length} retained envelopes. Clear envelopes first.`, + 409, + ); + if (nextLimit === maxEnvelopes) return; + await buffer.pipeline.setRetention({ maxEnvelopes: nextLimit }); + maxEnvelopes = nextLimit; + for (const listener of listeners) listener(getSettings()); + }); }, subscribeSettings(listener) { listeners.add(listener); diff --git a/packages/cli/src/commands/local/server.ts b/packages/cli/src/commands/local/server.ts index 88663ca79..9a0956b64 100644 --- a/packages/cli/src/commands/local/server.ts +++ b/packages/cli/src/commands/local/server.ts @@ -19,10 +19,11 @@ import { basename } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { serve } from "@hono/node-server"; import { - createSpotlightBuffer, - pushToSpotlightBuffer, -} from "@spotlightjs/spotlight/sdk"; -import { Hono } from "hono"; + localEnvelopeContentType, + localIngestStatus, + type LocalTelemetryBuffer, +} from "./telemetry-buffer.js"; +import { type Context, Hono } from "hono"; import { cors } from "hono/cors"; import { streamSSE } from "hono/streaming"; import type { SentryContext } from "../../context.js"; @@ -243,7 +244,7 @@ export function buildSSEHandler( } export function buildApp( - spotlightBuffer: ReturnType, + spotlightBuffer: LocalTelemetryBuffer, { uiActions = false, retention }: LocalReceiverOptions = {}, ): Hono { const app = new Hono(); @@ -361,7 +362,7 @@ export function buildApp( : undefined; try { validateMaxEnvelopes(value); - retention.resize(value); + await retention.resize(value); return c.json(retention.getSettings()); } catch (error) { if (error instanceof RetentionLimitError) @@ -400,14 +401,7 @@ export function buildApp( } }); - const ingest = async (c: { - req: { - arrayBuffer: () => Promise; - header: (name: string) => string | undefined; - query: (name: string) => string | undefined; - }; - body: (data: null, status: number) => Response; - }) => { + const ingest = async (c: Context) => { const contentLength = Number(c.req.header("content-length") ?? 0); if (contentLength > MAX_BODY_BYTES) { return c.body(null, 413); @@ -427,25 +421,19 @@ export function buildApp( ) { contentType = SENTRY_CONTENT_TYPE; } - const rawEncoding = c.req.header("content-encoding"); - const contentEncoding = ( - rawEncoding === "gzip" || - rawEncoding === "deflate" || - rawEncoding === "br" - ? rawEncoding - : undefined - ) as "gzip" | "deflate" | "br" | undefined; + const contentEncoding = c.req.header("content-encoding"); const userAgent = c.req.header("user-agent"); - pushToSpotlightBuffer({ - spotlightBuffer, + const receipt = await spotlightBuffer.ingestPayload({ body, encoding: contentEncoding, - contentType, + contentType: localEnvelopeContentType(contentType), userAgent, }); - return c.body(null, 204); + const status = localIngestStatus(receipt); + if (status === 503) c.header("Retry-After", "1"); + return c.body(null, status); }; app.post("/stream", ingest); diff --git a/packages/cli/src/commands/local/session-store.ts b/packages/cli/src/commands/local/session-store.ts index e1c393572..3f0c62e16 100644 --- a/packages/cli/src/commands/local/session-store.ts +++ b/packages/cli/src/commands/local/session-store.ts @@ -5,7 +5,7 @@ */ import { randomBytes } from "node:crypto"; -import { createSpotlightBuffer } from "@spotlightjs/spotlight/sdk"; +import type { LocalTelemetryBuffer } from "./telemetry-buffer.js"; import { createLocalRetentionBuffer, type LocalRetentionBuffer, @@ -35,7 +35,7 @@ export type LocalSession = { }; type StoredSession = LocalSession & { - buffer: ReturnType; + buffer: LocalTelemetryBuffer; retention: LocalRetentionBuffer; ingestCapability: string; streamCapability: string; @@ -62,7 +62,7 @@ export class AmbiguousLocalSessionError extends Error { /** In-memory session lifecycle and access to isolated telemetry buffers. */ export type LocalSessionStore = { create: (input: CreateLocalSessionInput) => LocalSession; - setMaxEnvelopes: (reference: string, value: number) => LocalSession; + setMaxEnvelopes: (reference: string, value: number) => Promise; getRetention: (reference: string) => LocalRetentionBuffer; close: (reference: string) => LocalSession; reset: (reference: string) => void; @@ -74,7 +74,7 @@ export type LocalSessionStore = { ingest: string; stream: string; }; - getBuffer: (reference: string) => ReturnType; + getBuffer: (reference: string) => LocalTelemetryBuffer; list: () => LocalSession[]; pruneExpired: () => string[]; recordActivity: (reference: string) => LocalSession; @@ -169,10 +169,12 @@ export function createLocalSessionStore({ getRetention(reference) { return resolveStored(reference).retention; }, - setMaxEnvelopes(reference, value) { + async setMaxEnvelopes(reference, value) { const session = resolveStored(reference); - session.retention.resize(value); - return toPublicSession(update(session, { maxEnvelopes: value })); + await session.retention.resize(value); + return toPublicSession( + update(resolveStored(reference), { maxEnvelopes: value }), + ); }, close(reference) { const session = resolveStored(reference); @@ -238,6 +240,7 @@ export function createLocalSessionStore({ if (session?.clientId && clientSessions.get(session.clientId) === id) { clientSessions.delete(session.clientId); } + session?.buffer.dispose(); sessions.delete(id); listeners.delete(id); } diff --git a/packages/cli/src/commands/local/telemetry-buffer.ts b/packages/cli/src/commands/local/telemetry-buffer.ts new file mode 100644 index 000000000..022d28d71 --- /dev/null +++ b/packages/cli/src/commands/local/telemetry-buffer.ts @@ -0,0 +1,252 @@ +/** HTTP/SSE compatibility facade over the shared telemetry pipeline. */ +import { randomUUID } from "node:crypto"; +import { Buffer } from "node:buffer"; +import { types } from "node:util"; +import { + createTelemetryPipeline, + type TelemetryPayload, + type TelemetryPipeline, +} from "@sentry/telemetry/node"; +import type { IgnoreReason, IngestResult } from "@sentry/telemetry"; +import { MAX_ENVELOPES } from "./retention-limits.js"; + +/** Parsed wire records consumed by the existing terminal and SSE formatters. */ +export type LocalEnvelope = { + envelope: [Record, [{ type?: string }, unknown][]]; +}; +export type LocalEnvelopeContainer = { + getParsedEnvelope(): LocalEnvelope; + getContentType(): string; + getData(): Buffer; + getEventTypes(): string[]; +}; +type ReadFilter = { all?: true; timeWindow?: number; envelopeId?: string }; +type Subscriber = { + callback: (container: LocalEnvelopeContainer) => void; + position: number; + scheduled?: NodeJS.Immediate; +}; +export type LocalTelemetryBuffer = { + pipeline: TelemetryPipeline; + ingestPayload( + payload: TelemetryPayload, + ): ReturnType; + read(filters?: ReadFilter): LocalEnvelopeContainer[]; + subscribe(callback: Subscriber["callback"], lastEventId?: string): string; + unsubscribe(id: string): void; + stream(id: string): void; + /** Serialize retention controls after submitted writes and before later admissions. */ + withWriteBarrier(operation: () => Promise): Promise; + drain(): Promise; + clear(): void; + reset(): void; + dispose(): void; +}; + +/** Legacy SDK beacon and raw clients post envelopes as plain text. */ +export function localEnvelopeContentType(value?: string): string { + const mediaType = value?.split(";")[0]?.trim().toLowerCase(); + return !mediaType || mediaType === "text/plain" + ? "application/x-sentry-envelope" + : (value ?? "application/x-sentry-envelope"); +} + +/** Convert SDK cancellation/admission outcomes into transport retry responses. */ +export function localIngestStatus(receipt: IngestResult): 204 | 400 | 503 { + if (receipt.status === "invalid") return 400; + if ( + receipt.status === "ignored" && + ["queue_full", "disposed", "reset", "aborted"].includes(receipt.reason) + ) + return 503; + return 204; +} + +/** Keep transport cursors and arrival times here; decoding and telemetry stay in the SDK. */ +export function createLocalTelemetryBuffer( + maxEnvelopes = 500, +): LocalTelemetryBuffer { + const maxPending = 64; + const pipeline = createTelemetryPipeline({ + maxPending, + retention: { maxEnvelopes }, + // Keep raw downloads available for the count retention range, within the + // same byte budget as the parsed telemetry store. + decodedHistory: { maxEntries: MAX_ENVELOPES, maxBytes: 64 * 1024 * 1024 }, + maxPayloadBytes: 10 * 1024 * 1024, + }); + const entries = new Map< + string, + { + container: LocalEnvelopeContainer; + arrivedAt: number; + position: number; + } + >(); + const subscribers = new Map(); + let position = 0; + let generation = 0; + let disposed = false; + let controls = 0; + let barrier: Promise = Promise.resolve(); + const writes = new Set>(); + function ignored(reason: IgnoreReason): IngestResult { + return { + status: "ignored", + reason, + retained: false, + revision: pipeline.getStats().revision, + diagnostics: [], + counts: { + events: 0, + spans: 0, + logs: 0, + skippedItems: 0, + duplicateEvents: 0, + }, + }; + } + function reset() { + // Clear capture data while preserving requested receiver retention configuration. + // Pending controls remain ordered; ingress from the previous generation is cancelled. + generation++; + writes.clear(); + pipeline.reset(); + } + const stream = (id: string) => { + const subscriber = subscribers.get(id); + if (!subscriber) return; + for (const entry of entries.values()) { + if (entry.position <= subscriber.position) continue; + subscriber.position = entry.position; + subscriber.callback(entry.container); + } + }; + const unsubscribePipeline = pipeline.subscribe((snapshot) => { + const retained = new Set(); + for (const envelope of snapshot.envelopes) { + const id = String(envelope[0].__spotlight_envelope_id); + retained.add(id); + if (entries.has(id)) continue; + const parsed: LocalEnvelope = { + envelope: [ + envelope[0], + envelope[1].map(([header, payload]) => [header, payload]), + ], + }; + entries.set(id, { + arrivedAt: Date.now(), + position: ++position, + container: { + getParsedEnvelope: () => parsed, + getContentType: () => "application/x-sentry-envelope", + getEventTypes: () => + parsed.envelope[1].flatMap(([header]) => + header.type ? [header.type] : [], + ), + getData: () => { + const payload = pipeline.getDecodedPayload(id); + if (!payload) + throw new Error("Raw envelope is no longer retained."); + return payload.body; + }, + }, + }); + } + for (const id of entries.keys()) if (!retained.has(id)) entries.delete(id); + for (const id of subscribers.keys()) stream(id); + }); + const unsubscribe = (id: string) => { + const subscriber = subscribers.get(id); + if (subscriber?.scheduled) clearImmediate(subscriber.scheduled); + subscribers.delete(id); + }; + return { + pipeline, + ingestPayload(payload) { + if (disposed) return Promise.resolve(ignored("disposed")); + if (writes.size >= maxPending) + return Promise.resolve(ignored("queue_full")); + const epoch = generation; + // A control operation blocks new admissions until its policy check and update commit. + // Copy deferred inputs now, matching standalone SDK ingestion semantics. + const captured = + controls > 0 + ? { + ...payload, + body: types.isUint8Array(payload.body) + ? Buffer.from(payload.body) + : payload.body, + } + : payload; + const operation = + controls === 0 + ? pipeline.ingestPayload(captured) + : barrier.then(() => { + if (disposed) return ignored("disposed"); + if (epoch !== generation) return ignored("reset"); + return pipeline.ingestPayload(captured); + }); + writes.add(operation); + const release = () => { + writes.delete(operation); + }; + void operation.then(release, release); + return operation; + }, + withWriteBarrier(operation) { + const preceding = [...writes]; + controls++; + const update = barrier.then(async () => { + await Promise.allSettled(preceding); + if (!disposed) await operation(); + }); + barrier = update.then( + () => {}, + () => {}, + ); + const release = () => { + controls--; + }; + void update.then(release, release); + return update; + }, + async drain() { + await Promise.allSettled([...writes, barrier, pipeline.drain()]); + }, + read(filters = { timeWindow: 60 }) { + return [...entries.entries()] + .reverse() + .filter( + ([id, entry]) => + (filters.envelopeId === undefined || id === filters.envelopeId) && + (filters.timeWindow === undefined || + entry.arrivedAt > Date.now() - filters.timeWindow * 1000), + ) + .map(([, entry]) => entry.container); + }, + subscribe(callback, lastEventId) { + const id = randomUUID(); + const subscriber: Subscriber = { + callback, + position: entries.get(lastEventId ?? "")?.position ?? 0, + }; + subscribers.set(id, subscriber); + subscriber.scheduled = setImmediate(() => stream(id)); + subscriber.scheduled.unref(); + return id; + }, + unsubscribe, + stream, + clear: reset, + reset, + dispose() { + disposed = true; + generation++; + for (const id of subscribers.keys()) unsubscribe(id); + unsubscribePipeline(); + pipeline.dispose(); + entries.clear(); + }, + }; +} diff --git a/packages/cli/src/lib/init/verify-setup.ts b/packages/cli/src/lib/init/verify-setup.ts index e77a4bd22..ab94862c5 100644 --- a/packages/cli/src/lib/init/verify-setup.ts +++ b/packages/cli/src/lib/init/verify-setup.ts @@ -19,7 +19,7 @@ import { captureException, setTag, } from "@sentry/node-core/light"; -import { createSpotlightBuffer } from "@spotlightjs/spotlight/sdk"; +import { createLocalTelemetryBuffer } from "../../commands/local/telemetry-buffer.js"; import { BUFFER_SIZE, shutdownServer } from "../../commands/local/run.js"; import { buildApp, tryListen } from "../../commands/local/server.js"; import { detectDevCommand } from "../dev-script.js"; @@ -377,7 +377,7 @@ export async function verifySetup( logger.debug(`Verification command: ${detected.args.join(" ")}`); - const buffer = createSpotlightBuffer(BUFFER_SIZE); + const buffer = createLocalTelemetryBuffer(BUFFER_SIZE); const app = buildApp(buffer); let server: Awaited>["server"]; diff --git a/packages/cli/test/commands/local/run.test.ts b/packages/cli/test/commands/local/run.test.ts index abfc8ed82..ec4d29c8f 100644 --- a/packages/cli/test/commands/local/run.test.ts +++ b/packages/cli/test/commands/local/run.test.ts @@ -7,7 +7,7 @@ import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { createSpotlightBuffer } from "@spotlightjs/spotlight/sdk"; +import { createLocalTelemetryBuffer } from "../../../src/commands/local/telemetry-buffer.js"; import { Hono } from "hono"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { @@ -447,7 +447,7 @@ describe("sentry local run", () => { // Reproduces the case where the Spotlight desktop app (or another // `sentry local serve`) already owns the port: `run` must attach as an SSE // consumer instead of going silent for the whole session. - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const { server, port } = await tryListen(buildApp(buffer), 0, "127.0.0.1"); // preload.ts mocks fetch to block external calls; this test talks to a @@ -499,7 +499,7 @@ describe("sentry local run", () => { test("writes normalized NDJSON to stdout when following an existing server", async () => { // A one-command agent workflow uses `local run --format json`, including // when another process already owns the local receiver. - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const { server, port } = await tryListen(buildApp(buffer), 0, "127.0.0.1"); const savedFetch = globalThis.fetch; const realFetch = (globalThis as { __originalFetch?: typeof fetch }) diff --git a/packages/cli/test/commands/local/server.test.ts b/packages/cli/test/commands/local/server.test.ts index a1fc1c7a2..8513d3263 100644 --- a/packages/cli/test/commands/local/server.test.ts +++ b/packages/cli/test/commands/local/server.test.ts @@ -5,7 +5,7 @@ * feedSSELine, and parsePort. */ -import { createSpotlightBuffer } from "@spotlightjs/spotlight/sdk"; +import { createLocalTelemetryBuffer } from "../../../src/commands/local/telemetry-buffer.js"; import { createLocalControlPlane } from "../../../src/commands/local/control-plane.js"; import { createLocalRetentionBuffer } from "../../../src/commands/local/retention-buffer.js"; import { logger } from "../../../src/lib/logger.js"; @@ -263,7 +263,7 @@ describe("sentry local serve --open", () => { test("scoped serve opens the viewer with explicit session discovery", async () => { const daemon = await import("../../../src/commands/local/daemon.js"); const { server, port } = await tryListen( - buildApp(createSpotlightBuffer(5)), + buildApp(createLocalTelemetryBuffer(5)), 0, "127.0.0.1", ); @@ -346,7 +346,7 @@ describe("sentry local serve --open", () => { } globalThis.fetch = realFetch; const { server, port } = await tryListen( - buildApp(createSpotlightBuffer(10)), + buildApp(createLocalTelemetryBuffer(10)), 0, "127.0.0.1", ); @@ -486,7 +486,7 @@ const TEST_ENVELOPE = * otherwise the subscriber misses it and the read hangs. */ async function readFirstSSEEvent(streamPath: string): Promise { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request(streamPath, { @@ -651,7 +651,7 @@ describe("buildApp", () => { }); test("health endpoint returns OK", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/health"); @@ -660,7 +660,7 @@ describe("buildApp", () => { }); test("advertises session-only UI capabilities", async () => { - const app = buildApp(createSpotlightBuffer(10), { uiActions: true }); + const app = buildApp(createLocalTelemetryBuffer(10), { uiActions: true }); const res = await app.request("/capabilities"); @@ -672,7 +672,7 @@ describe("buildApp", () => { }); test("clears buffered envelopes and exposes a retained raw envelope", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer, { uiActions: true }); await app.request("/stream", { method: "POST", @@ -694,7 +694,7 @@ describe("buildApp", () => { }); test("ingest endpoint accepts envelopes and returns 204", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const envelope = @@ -708,7 +708,7 @@ describe("buildApp", () => { }); test("ingest via /api/:projectId/envelope/ returns 204", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const envelope = @@ -722,7 +722,7 @@ describe("buildApp", () => { }); test("ingest rejects oversized payloads with 413", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/stream", { @@ -737,7 +737,7 @@ describe("buildApp", () => { }); test("CORS allows localhost origins", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/health", { @@ -749,7 +749,7 @@ describe("buildApp", () => { }); test("CORS allows the hosted local UI to read the event stream", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/stream", { @@ -764,7 +764,7 @@ describe("buildApp", () => { }); test("CORS allows the configured Sentry Local preview to read the event stream", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const origin = "https://sentry-local-git-codex-featlocal-observability-workspace.sentry.dev"; @@ -779,7 +779,7 @@ describe("buildApp", () => { }); test("CORS allows HTTPS Sentry Local preview branches to read the event stream", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const origin = "https://sentry-local-git-feature-branch.sentry.dev"; @@ -794,7 +794,7 @@ describe("buildApp", () => { }); test("CORS blocks unrelated Sentry-hosted origins from reading the event stream", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/stream", { @@ -808,7 +808,7 @@ describe("buildApp", () => { }); test("CORS permits the SSE resume header from the hosted local UI", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/stream", { @@ -832,7 +832,7 @@ describe("buildApp", () => { }); test("rejects hosted-origin writes to the local receiver", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/stream", { @@ -848,7 +848,7 @@ describe("buildApp", () => { }); test("rejects hosted-origin receiver controls", async () => { - const app = buildApp(createSpotlightBuffer(10), { uiActions: true }); + const app = buildApp(createLocalTelemetryBuffer(10), { uiActions: true }); const res = await app.request("/capabilities", { headers: { Origin: "https://local.sentry.dev" }, @@ -858,7 +858,7 @@ describe("buildApp", () => { }); test("does not expose receiver controls without loopback authorization", async () => { - const app = buildApp(createSpotlightBuffer(10)); + const app = buildApp(createLocalTelemetryBuffer(10)); expect((await app.request("/capabilities")).status).toBe(403); expect((await app.request("/clear", { method: "DELETE" })).status).toBe( @@ -867,7 +867,7 @@ describe("buildApp", () => { }); test("CORS blocks lookalike hosted UI origins", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/health", { @@ -877,7 +877,7 @@ describe("buildApp", () => { }); test("CORS requires HTTPS for hosted Sentry preview origins", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/stream", { @@ -893,7 +893,7 @@ describe("buildApp", () => { }); test("limits the configured Sentry Local preview to stream reads", async () => { - const app = buildApp(createSpotlightBuffer(10), { uiActions: true }); + const app = buildApp(createLocalTelemetryBuffer(10), { uiActions: true }); const origin = "https://sentry-local-git-codex-featlocal-observability-workspace.sentry.dev"; @@ -914,7 +914,7 @@ describe("buildApp", () => { }); test("CORS blocks non-localhost origins", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/health", { @@ -924,7 +924,7 @@ describe("buildApp", () => { }); test("SSE stream endpoint returns event-stream content type", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/stream", { @@ -939,7 +939,7 @@ describe("buildApp", () => { }); test("advertises itself as a Spotlight sidecar via X-Powered-By", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/health"); @@ -950,7 +950,7 @@ describe("buildApp", () => { }); test("sets X-Powered-By on ingest responses too", async () => { - const buffer = createSpotlightBuffer(10); + const buffer = createLocalTelemetryBuffer(10); const app = buildApp(buffer); const res = await app.request("/stream", { diff --git a/packages/cli/test/commands/local/session-store.test.ts b/packages/cli/test/commands/local/session-store.test.ts index dd54517df..2f68715be 100644 --- a/packages/cli/test/commands/local/session-store.test.ts +++ b/packages/cli/test/commands/local/session-store.test.ts @@ -1,9 +1,8 @@ -import { pushToSpotlightBuffer } from "@spotlightjs/spotlight/sdk"; import { describe, expect, test, vi } from "vitest"; import { createLocalSessionStore } from "../../../src/commands/local/session-store.js"; describe("LocalSessionStore", () => { - test("grows retention without losing envelopes, IDs or live subscribers", () => { + test("grows retention without losing envelopes, IDs or live subscribers", async () => { const store = createLocalSessionStore(); const session = store.create({ label: "web", @@ -15,9 +14,8 @@ describe("LocalSessionStore", () => { const reader = buffer.subscribe((container) => messages.push(container.getData().toString()), ); - function ingest(id: string) { - pushToSpotlightBuffer({ - spotlightBuffer: buffer, + async function ingest(id: string) { + await buffer.ingestPayload({ contentType: "application/x-sentry-envelope", body: Buffer.from( `{"event_id":"${id}"}\n{"type":"event"}\n{"message":"${id}"}`, @@ -25,31 +23,31 @@ describe("LocalSessionStore", () => { }); buffer.stream(reader); } - ingest("one"); - ingest("two"); + await ingest("one"); + await ingest("two"); const retained = buffer.read({ all: true }); expect(store.resolve(session.id).maxEnvelopes).toBe(2); - store.setMaxEnvelopes(session.id, 4); + await store.setMaxEnvelopes(session.id, 4); expect(store.getBuffer(session.id)).toBe(buffer); expect(buffer.read({ all: true })).toEqual(retained); - ingest("three"); - ingest("four"); + await ingest("three"); + await ingest("four"); expect(buffer.read({ all: true })).toHaveLength(4); expect(messages).toHaveLength(4); - expect(() => store.setMaxEnvelopes(session.id, 3)).toThrow( + await expect(store.setMaxEnvelopes(session.id, 3)).rejects.toThrow( /retained envelopes/, ); expect(buffer.read({ all: true })).toHaveLength(4); buffer.clear(); - store.setMaxEnvelopes(session.id, 1); - ingest("five"); - ingest("six"); + await store.setMaxEnvelopes(session.id, 1); + await ingest("five"); + await ingest("six"); expect(buffer.read({ all: true })).toHaveLength(1); expect(messages).toHaveLength(6); buffer.unsubscribe(reader); }); - test("keeps original arrival times when resizing", () => { + test("keeps original arrival times when resizing", async () => { const store = createLocalSessionStore(); const session = store.create({ label: "timed", @@ -59,13 +57,12 @@ describe("LocalSessionStore", () => { const buffer = store.getBuffer(session.id); const time = vi.spyOn(Date, "now").mockReturnValue(1000); try { - pushToSpotlightBuffer({ - spotlightBuffer: buffer, + await buffer.ingestPayload({ contentType: "application/x-sentry-envelope", body: Buffer.from('{"event_id":"old"}\n{"type":"event"}\n{}'), }); time.mockReturnValue(61001); - store.setMaxEnvelopes(session.id, 4); + await store.setMaxEnvelopes(session.id, 4); expect(buffer.read({ timeWindow: 60 })).toHaveLength(0); expect(buffer.read({ all: true })).toHaveLength(1); } finally { diff --git a/packages/cli/test/commands/local/telemetry-buffer.test.ts b/packages/cli/test/commands/local/telemetry-buffer.test.ts new file mode 100644 index 000000000..04985d6e8 --- /dev/null +++ b/packages/cli/test/commands/local/telemetry-buffer.test.ts @@ -0,0 +1,473 @@ +import { runInNewContext } from "node:vm"; +import { gzipSync, deflateSync, brotliCompressSync } from "node:zlib"; +import { describe, expect, test } from "vitest"; +import { createLocalTelemetryBuffer } from "../../../src/commands/local/telemetry-buffer.js"; +import { + buildApp, + buildSSEHandler, +} from "../../../src/commands/local/server.js"; +import { createLocalControlPlane } from "../../../src/commands/local/control-plane.js"; +import { SENTRY_CONTENT_TYPE } from "../../../src/lib/formatters/local.js"; + +const event = (id: string) => + `{"event_id":"${id}"}\n{"type":"event"}\n{"message":"hello"}`; + +describe("shared telemetry receiver", () => { + test("accepts streamed span v2 through the shared pipeline", async () => { + const buffer = createLocalTelemetryBuffer(); + try { + const app = buildApp(buffer); + const response = await app.request("/api/42/envelope/", { + method: "POST", + headers: { "Content-Type": SENTRY_CONTENT_TYPE }, + body: + '{}\n{"type":"span"}\n' + + JSON.stringify({ + version: 2, + items: [ + { + trace_id: "a".repeat(32), + span_id: "b".repeat(16), + name: "SELECT 1", + start_timestamp: 1000, + end_timestamp: 1001, + attributes: { "sentry.op": { type: "string", value: "db" } }, + }, + ], + }), + }); + expect(response.status).toBe(204); + expect(buffer.pipeline.getSpans()).toHaveLength(1); + expect( + buffer.read({ all: true })[0]?.getParsedEnvelope().envelope[1][0]?.[0] + .type, + ).toBe("span"); + } finally { + buffer.dispose(); + } + }); + + test.each([ + "not-json", + '{}\n{"type":"event"}\nnot-json', + '{}\n{"type":"attachment","length":99}\nshort', + ])("rejects malformed input without retaining it: %s", async (body) => { + const buffer = createLocalTelemetryBuffer(); + try { + const response = await buildApp(buffer).request("/stream", { + method: "POST", + headers: { "Content-Type": SENTRY_CONTENT_TYPE }, + body, + }); + expect(response.status).toBe(400); + expect(buffer.read({ all: true })).toEqual([]); + } finally { + buffer.dispose(); + } + }); + + test.each([ + ["gzip", gzipSync], + ["deflate", deflateSync], + ["br", brotliCompressSync], + ] as const)( + "retains original decoded bytes after %s compression", + async (encoding, compress) => { + const buffer = createLocalTelemetryBuffer(); + try { + const body = Buffer.from(event("raw")); + const app = buildApp(buffer, { uiActions: true }); + const response = await app.request("/stream", { + method: "POST", + headers: { + "Content-Type": SENTRY_CONTENT_TYPE, + "Content-Encoding": encoding, + }, + body: new Uint8Array(compress(body)), + }); + expect(response.status).toBe(204); + const container = buffer.read({ all: true })[0]!; + const id = String( + container.getParsedEnvelope().envelope[0].__spotlight_envelope_id, + ); + const raw = await app.request(`/envelope/${id}`); + expect(Buffer.from(await raw.arrayBuffer())).toEqual(body); + } finally { + buffer.dispose(); + } + }, + ); + + test.each([undefined, "text/plain; charset=UTF-8"])( + "accepts legacy plain envelopes with media type %s", + async (contentType) => { + const buffer = createLocalTelemetryBuffer(); + try { + const response = await buildApp(buffer).request("/stream", { + method: "POST", + headers: contentType ? { "Content-Type": contentType } : {}, + body: event("plain"), + }); + expect(response.status).toBe(204); + expect(buffer.read({ all: true })).toHaveLength(1); + } finally { + buffer.dispose(); + } + }, + ); + + test("rejects invalid compressed bodies without retaining them", async () => { + const buffer = createLocalTelemetryBuffer(); + try { + const response = await buildApp(buffer).request("/stream", { + method: "POST", + headers: { + "Content-Type": SENTRY_CONTENT_TYPE, + "Content-Encoding": "gzip", + }, + body: "not gzip", + }); + expect(response.status).toBe(400); + expect(buffer.read({ all: true })).toEqual([]); + } finally { + buffer.dispose(); + } + }); + + test("delivers an admitted compressed envelope before the final session close frame", async () => { + const plane = createLocalControlPlane({ controlToken: "test" }); + const session = plane.sessionStore.create({ + label: "closing", + cwd: "/test", + }); + const buffer = plane.sessionStore.getBuffer(session.id); + const capability = plane.sessionStore.getCapabilities(session.id).stream; + const response = await plane.app.request( + `/_local/v1/sessions/${session.id}/stream?cap=${capability}`, + ); + const reader = response.body!.getReader(); + try { + await reader.read(); // Initial open-session metadata. + const ingest = buffer.ingestPayload({ + body: gzipSync(event("before-close")), + encoding: "gzip", + contentType: SENTRY_CONTENT_TYPE, + }); + plane.sessionStore.close(session.id); + await ingest; + let text = ""; + const decoder = new TextDecoder(); + for (;;) { + const next = await reader.read(); + if (next.done) break; + text += decoder.decode(next.value); + } + expect(text).toContain("before-close"); + expect(text.indexOf("before-close")).toBeLessThan( + text.indexOf('"state":"retained"'), + ); + } finally { + await reader.cancel(); + buffer.dispose(); + } + }); + + test("keeps opaque attachment bytes and SSE base64 transport compatible", async () => { + const buffer = createLocalTelemetryBuffer(); + try { + const bytes = Buffer.from([0, 255, 10, 128]); + const body = Buffer.concat([ + Buffer.from( + '{}\n{"type":"attachment","length":4,"content_type":"application/octet-stream"}\n', + ), + bytes, + ]); + await buffer.ingestPayload({ body, contentType: SENTRY_CONTENT_TYPE }); + const container = buffer.read({ all: true })[0]!; + expect(container.getData()).toEqual(body); + const frames: { event?: string; data: string }[] = []; + buildSSEHandler( + { + writeSSE: async (frame) => { + frames.push(frame); + }, + }, + true, + )(container); + expect(frames[0]?.event).toBe(`${SENTRY_CONTENT_TYPE};base64`); + const parsed = JSON.parse(frames[0]!.data); + expect(parsed[1][0][1].data).toBe(bytes.toString("base64")); + } finally { + buffer.dispose(); + } + }); + + test("replays after cursors, falls back for stale cursors, and keeps subscribers after clear", async () => { + const buffer = createLocalTelemetryBuffer(2); + try { + for (const id of ["first", "second", "third"]) + await buffer.ingestPayload({ + body: event(id), + contentType: SENTRY_CONTENT_TYPE, + }); + const history = buffer.read({ all: true }); + const secondId = String( + history[1]!.getParsedEnvelope().envelope[0].__spotlight_envelope_id, + ); + const seen: string[] = []; + const reader = buffer.subscribe( + (item) => + seen.push(String(item.getParsedEnvelope().envelope[0].event_id)), + secondId, + ); + buffer.stream(reader); + expect(seen).toEqual(["third"]); + const stale: string[] = []; + const staleReader = buffer.subscribe( + (item) => + stale.push(String(item.getParsedEnvelope().envelope[0].event_id)), + "unknown", + ); + buffer.stream(staleReader); + expect(stale).toEqual(["second", "third"]); + buffer.clear(); + await buffer.ingestPayload({ + body: event("fourth"), + contentType: SENTRY_CONTENT_TYPE, + }); + expect(seen).toEqual(["third", "fourth"]); + expect(buffer.pipeline.getEnvelopes()).toHaveLength(1); + } finally { + buffer.dispose(); + } + }); +}); + +test.each(["global", "scoped"] as const)( + "%s receiver reports backpressure instead of acknowledging discarded burst deliveries", + async (scope) => { + const plane = createLocalControlPlane({ controlToken: "burst" }); + const session = plane.sessionStore.create({ + label: "burst", + cwd: "/burst", + }); + const buffer = + scope === "global" + ? createLocalTelemetryBuffer() + : plane.sessionStore.getBuffer(session.id); + const app = scope === "global" ? buildApp(buffer) : plane.app; + const path = + scope === "global" + ? "/stream" + : `/_local/v1/sessions/${session.id}/ingest/${plane.sessionStore.getCapabilities(session.id).ingest}`; + try { + const responses = await Promise.all( + Array.from( + { length: 100 }, + async (_, index) => + await app.request(path, { + method: "POST", + headers: { "Content-Type": SENTRY_CONTENT_TYPE }, + body: event(`burst-${index}`), + }), + ), + ); + const accepted = responses.filter((response) => response.status === 204); + const retry = responses.filter((response) => response.status === 503); + expect(accepted.length + retry.length).toBe(100); + expect(retry.length).toBeGreaterThan(0); + expect( + retry.every((response) => response.headers.get("Retry-After") === "1"), + ).toBe(true); + expect(buffer.read({ all: true })).toHaveLength(accepted.length); + } finally { + buffer.dispose(); + if (scope === "global") + plane.sessionStore.getBuffer(session.id).dispose(); + } + }, +); + +test("retention resize checks admitted compressed ingress before refusing destructive reductions", async () => { + const { createLocalRetentionBuffer } = + await import("../../../src/commands/local/retention-buffer.js"); + const retention = createLocalRetentionBuffer(3); + const buffer = retention.buffer; + try { + await buffer.ingestPayload({ body: event("before") }); + const admitted = buffer.ingestPayload({ + body: gzipSync(event("during")), + encoding: "gzip", + }); + const resize = retention.resize(1); + const later = buffer.ingestPayload({ body: event("after") }); + await expect(resize).rejects.toMatchObject({ status: 409 }); + await Promise.all([admitted, later]); + expect(retention.getSettings()).toEqual({ + maxEnvelopes: 3, + retainedEnvelopes: 3, + }); + expect( + buffer + .read({ all: true }) + .map((item) => item.getParsedEnvelope().envelope[0].event_id), + ).toEqual(["after", "during", "before"]); + } finally { + buffer.dispose(); + } +}); + +test("close waits for writes admitted behind a retention resize and keeps the session closed", async () => { + const plane = createLocalControlPlane({ controlToken: "resize-close" }); + const session = plane.sessionStore.create({ + label: "resize-close", + cwd: "/test", + maxEnvelopes: 3, + }); + const buffer = plane.sessionStore.getBuffer(session.id); + const capability = plane.sessionStore.getCapabilities(session.id).stream; + const response = await plane.app.request( + `/_local/v1/sessions/${session.id}/stream?cap=${capability}`, + ); + const reader = response.body!.getReader(); + try { + await reader.read(); + const first = buffer.ingestPayload({ + body: gzipSync(event("before-resize")), + encoding: "gzip", + }); + const resize = plane.sessionStore.setMaxEnvelopes(session.id, 4); + const deferred = buffer.ingestPayload({ body: event("after-resize") }); + plane.sessionStore.close(session.id); + await Promise.all([first, resize, deferred]); + expect(plane.sessionStore.resolve(session.id).state).toBe("retained"); + let text = ""; + const decoder = new TextDecoder(); + for (;;) { + const next = await reader.read(); + if (next.done) break; + text += decoder.decode(next.value); + } + expect(text).toContain("before-resize"); + expect(text).toContain("after-resize"); + expect(text.indexOf("after-resize")).toBeLessThan( + text.indexOf('"state":"retained"'), + ); + } finally { + await reader.cancel(); + buffer.dispose(); + } +}); + +test("clear cancels writes deferred behind retention controls and accepts new writes", async () => { + const { createLocalRetentionBuffer } = + await import("../../../src/commands/local/retention-buffer.js"); + const retention = createLocalRetentionBuffer(3); + const buffer = retention.buffer; + try { + const first = buffer.ingestPayload({ + body: gzipSync(event("before-clear")), + encoding: "gzip", + }); + const resize = retention.resize(4); + const deferred = buffer.ingestPayload({ + body: event("deferred-before-clear"), + }); + buffer.clear(); + const fresh = buffer.ingestPayload({ body: event("after-clear") }); + await resize; + expect(retention.getSettings().maxEnvelopes).toBe(4); + expect(await first).toMatchObject({ status: "ignored", reason: "reset" }); + expect(await deferred).toMatchObject({ + status: "ignored", + reason: "reset", + }); + expect(await fresh).toMatchObject({ status: "processed" }); + await buffer.drain(); + expect( + buffer + .read({ all: true }) + .map((item) => item.getParsedEnvelope().envelope[0].event_id), + ).toEqual(["after-clear"]); + } finally { + buffer.dispose(); + } +}); + +test("receiver drain excludes retention controls submitted after the barrier", async () => { + const buffer = createLocalTelemetryBuffer(); + let release!: () => void; + let releaseLater!: () => void; + const firstGate = new Promise((resolve) => { + release = resolve; + }); + const laterGate = new Promise((resolve) => { + releaseLater = resolve; + }); + try { + const first = buffer.withWriteBarrier(async () => { + await firstGate; + }); + const ingress = buffer.ingestPayload({ body: event("earlier") }); + const drained = buffer.drain(); + const later = buffer.withWriteBarrier(async () => { + await laterGate; + }); + release(); + await drained; + expect((await ingress).status).toBe("processed"); + releaseLater(); + await Promise.all([first, later]); + } finally { + release(); + releaseLater(); + buffer.dispose(); + } +}); + +test("clear releases admission capacity immediately for the new capture generation", async () => { + const buffer = createLocalTelemetryBuffer(); + try { + const old = Array.from({ length: 64 }, (_, index) => + buffer.ingestPayload({ body: event(`old-${index}`) }), + ); + buffer.clear(); + const fresh = buffer.ingestPayload({ body: event("fresh") }); + expect(await fresh).toMatchObject({ status: "processed" }); + expect( + (await Promise.all(old)).every( + (receipt) => receipt.status === "ignored" && receipt.reason === "reset", + ), + ).toBe(true); + expect(buffer.read({ all: true })).toHaveLength(1); + } finally { + buffer.dispose(); + } +}); + +test("copies cross-realm payload bytes before deferring them behind retention controls", async () => { + const buffer = createLocalTelemetryBuffer(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + try { + const control = buffer.withWriteBarrier(async () => { + await gate; + }); + const body = runInNewContext("Uint8Array.from(bytes)", { + bytes: [...Buffer.from(event("vm-copied"))], + }) as Uint8Array; + const ingress = buffer.ingestPayload({ body }); + body.fill(0); + release(); + await control; + expect(await ingress).toMatchObject({ status: "processed" }); + expect( + buffer.read({ all: true })[0]?.getParsedEnvelope().envelope[0].event_id, + ).toBe("vm-copied"); + } finally { + release(); + buffer.dispose(); + } +}); diff --git a/packages/telemetry/LICENSE b/packages/telemetry/LICENSE new file mode 100644 index 000000000..ceb7742a1 --- /dev/null +++ b/packages/telemetry/LICENSE @@ -0,0 +1,13 @@ +Copyright 2023 Sentry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/packages/telemetry/README.md b/packages/telemetry/README.md new file mode 100644 index 000000000..183b75f88 --- /dev/null +++ b/packages/telemetry/README.md @@ -0,0 +1,205 @@ +# `@sentry/telemetry` + +This package shares Sentry envelope normalization, trace assembly, and insight calculations across +server and frontend consumers. Each instance owns its state; feed server and +frontend instances the same envelope stream and match retention settings when +comparing their retained histories. + +## Choose an entry point + +| Entry | Factory | Input | +| ------------------------ | --------------------------- | ------------------------------------------------------------------ | +| `@sentry/telemetry` | `createTelemetryStore()` | Parsed Sentry envelopes in portable runtimes | +| `@sentry/telemetry/node` | `createTelemetryPipeline()` | Adds serialized bodies, decompression, and decoded payload history | + +## Ingest and read telemetry + +Use `@sentry/telemetry` for parsed Sentry envelopes in a browser, worker, or Node application. Use `@sentry/telemetry/node` for serialized or compressed payloads on Node 20+. Both expose the same processing and read APIs; creating an instance does not start a server. The portable entry has no Node or React dependency. + +```ts +import { createTelemetryStore, serializeSnapshot } from "@sentry/telemetry"; + +const telemetry = createTelemetryStore(); +const receipt = await telemetry.ingestEnvelope(parsedEnvelope, { + deliveryId: "transport-request-123", // Reuse for retries of this delivery. + signal: abortController.signal, +}); +if (receipt.status === "processed") { + console.log(receipt.envelopeId, receipt.counts, receipt.retained); +} else if (receipt.status === "ignored") { + console.log(receipt.reason); // For example: duplicate, aborted, queue_full. +} else { + console.log(receipt.diagnostics); // Literal codes, severity, optional itemIndex/spanIndex. +} +await telemetry.drain(); +const json = JSON.stringify(serializeSnapshot(telemetry.getSnapshot())); +telemetry.dispose(); +``` + +The Node adapter accepts `string`, `Uint8Array`, and `Buffer` bodies. `ingestPayload({ body, contentType?, encoding?, userAgent?, deliveryId?, signal? })` decodes `application/x-sentry-envelope` (the default content type) and supports identity, gzip, deflate, and Brotli encoding. Use `getDecodedPayload(envelopeId)` for a copied decompressed payload when it remains in the independently bounded history. Parsed envelopes still use `ingestEnvelope`; the payload methods are `ingestPayload` and `ingestPayloads`. + +```ts +import { createTelemetryPipeline } from "@sentry/telemetry/node"; + +const pipeline = createTelemetryPipeline({ + decodedHistory: { maxEntries: 100 }, +}); +const receipt = await pipeline.ingestPayload({ + body: envelopeBody, + deliveryId: "request-1", +}); +if (receipt.status === "processed") { + const payload = pipeline.getDecodedPayload(receipt.envelopeId); // Defensive decoded-byte copy. + const queries = pipeline.getDatabaseQuerySummaries({ traceId }); // Compact by default. +} +await pipeline.drain(); +pipeline.dispose(); +``` + +### Receipts, retries, and lifecycle + +Every input returns an immutable receipt with `status`, `diagnostics`, `revision`, `retained`, and `counts`: + +| Field | Meaning | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `status: "processed"` | A supported envelope was processed. This does not imply new telemetry or continued retention. | +| `status: "ignored"` | `reason` explains the outcome: `duplicate`, `unsupported_items`, `unsupported_content_type`, `reset`, `disposed`, `aborted`, or `queue_full`. | +| `status: "invalid"` | Diagnostic codes explain invalid data or exceeded payload limits. | +| `retained` | Whether this envelope is in processed retention when the receipt is produced. An unsupported envelope can be retained; an immediately evicted processed envelope is not. This says nothing about Node decoded history. | +| `revision` | Published snapshot revision at the outcome. A retry that changes no state need not advance it. | +| `counts.events`, `counts.spans`, `counts.logs` | Newly added event IDs, trace/span pairs, and log IDs during processing, before retention eviction. These overlap: standalone spans can also become events. They are not retained collection sizes. | +| `counts.skippedItems` | Unsupported wire items plus malformed entries skipped inside supported span containers. | +| `counts.duplicateEvents` | Candidate event observations that did not add a new event ID. Delivery-level duplicate receipts have zero counts. | + +Branch on diagnostic `code` and `severity`, or ignored `reason`, rather than message text. Diagnostics can accompany successful partial processing. `itemIndex` and `spanIndex`, when present, are zero-based locations. + +Delivery IDs deduplicate only while the corresponding envelope is retained. Eviction and `reset()` permit that delivery again. There is no content hashing: separately allocated envelopes without IDs are separate observations. Reusing the same parsed object or its explicit Spotlight envelope ID preserves its delivery identity within the retained window. Event-ID deduplication is separate from delivery deduplication. + +`ingestEnvelopes(iterable, { signal? })` returns receipts in input order and preserves valid neighbors of invalid inputs. It is not a transaction. `ingestPayloads(inputs)` does the same for Node inputs; each input carries its own signal and delivery ID. A batch occupies one queued operation, including while active. Items are copied as they are reached rather than copying the entire batch up front: keep queued batch items unchanged until their receipts resolve. Individual `ingestEnvelope` and `ingestPayload` calls copy their input before the first await. Cancellation returns `ignored/aborted`. `drain()` waits for operations already submitted at the call, excluding future work. `getStats()` reports pending operations, retained envelope count/bytes, revision, and disposal. `subscribe((snapshot, change) => ...)` returns an unsubscribe function; `change` identifies ingest, eviction, reset, or disposal and affected envelope IDs. Release timers and listeners with `dispose()`. + +An optional `enrichStacktrace(event, { signal })` hook receives a copied event. Observe its signal to cancel external work. A timeout reports a warning and keeps telemetry; abort, reset, disposal, or timeout prevent late hook writes from entering published state. + +### Read APIs and JSON + +Accept a `TelemetryReader` from `/sdk/core` in consumers that only query telemetry. Both factories implement it. `createTelemetryReader(getSnapshot)` also constructs a reader over an existing snapshot source. Singular getters such as `getTrace(id)` and `getEvent(id)` return `undefined` when absent; collection getters return readonly arrays. All processed timestamps and durations use milliseconds; time bounds are inclusive Unix milliseconds. + +```ts +const queries = reader.getDatabaseQuerySummaries({ + traceId, + fromMs, + toMs, + limit: 10, +}); +const detailed = reader.getDatabaseQuerySummaries({ + traceId, + includeSpans: true, +}); +const first = reader.readPage("events", { traceId, limit: 100, order: "desc" }); +if (first.status === "ok" && first.nextCursor) { + const next = reader.readPage("events", { + traceId, + limit: 100, + order: "desc", + cursor: first.nextCursor, + }); + // Handle next.status === "invalid" before using next.items. +} +``` + +Queries and resource summaries omit full span lists by default; opt in with `includeSpans: true`. Getters accept `fromMs`, `toMs`, `order` (`asc` by default), and a positive `limit`; getters have no implicit limit. Chronological ties use stable IDs. Summary rows sort by description and apply time bounds to their input spans. Invalid getter options throw `RangeError`. + +`readPage` supports `events`, `traces`, `logs`, and `spans`, defaults to 100 rows, and caps a page at 1000. It returns `status: "ok"`, `revision`, `items`, `total`, and optional `nextCursor`, or `status: "invalid"` with `invalid_query`, `invalid_cursor`, or `stale_cursor`. Cursors bind the collection, filters, order, limit, and snapshot revision. Keep the query unchanged between pages and restart after a stale cursor. `spanId` filtering on pages is supported for events only. Treat cursors as opaque. + +Snapshots preserve readonly maps and sets for in-process use. For JSON, use `serializeSnapshot(snapshot)` or `serializeTrace(trace)`: the schema version is `1`, maps become entry arrays, sets become value arrays, and span trees use IDs and `childSpanIds` instead of repeated object graphs. Unsupported non-wire values or unrelated cycles throw `TypeError`. These DTOs are export formats, not inputs to `ingestEnvelope`. Equivalent reads within the current revision reuse cached results; older snapshots remain stable after subsequent ingestion. + +Use `serializeSummaries(reader.getProfileFunctionSummaries())` or pass compact query/resource summaries to export `{ schemaVersion: 1, summaries }`. Profile `traceIds` sets become string arrays. Summaries requested with `includeSpans: true` are rejected at both the type and runtime boundaries; export the complete span graph with `serializeTrace` instead. + +### Limits and supported formats + +| Option | Default | +| ------------------------------------------------------------ | ------------------------------------------------------------------ | +| `maxPending` (ingestion operations, including active work) | 64 | +| `maxEnvelopeBytes` (serialized parsed envelope) | 32 MiB | +| `enrichmentTimeoutMs` | 5000 ms | +| `retention.maxEnvelopes` / `retention.maxBytes` | 500 / 64 MiB | +| `retention.maxAgeMs` | No age limit; when configured, expires automatically since receipt | +| Node `maxPayloadBytes` / `maxDecodedBytes` | 8 MiB / 32 MiB | +| Node `decodedHistory.maxEntries` / `decodedHistory.maxBytes` | 500 / 32 MiB | + +`setRetention({ maxEnvelopes?, maxBytes?, maxAgeMs? })` updates only supplied retention limits and resolves after queued eviction. Existing subscriptions and Node decoded history remain available. Age limits cannot be cleared; disposed instances ignore updates. Node updates follow earlier payload/parsed work, and `drain()` includes retention updates submitted before the drain. + +Limits must be positive safe integers; Node `decodedHistory.maxEntries` and `decodedHistory.maxBytes` also allow zero to disable decoded history. Byte limits use bytes, with 1 MiB = 1024 × 1024 bytes. The Node decoder bounds decompression output. Bound transport collection separately before passing a complete body to the SDK, as shown in the HTTP example. Retention evicts whole envelopes and the telemetry derived from them; the byte budget measures the serialized retained envelopes, not total JavaScript heap size. + +`SDK_CAPABILITIES` exposes readonly machine-readable metadata for legacy and v2 spans, profile v1/v2, supported item types, and timestamp units. Supported item types include event/error, transaction, span, log, profile/profile_chunk, feedback, and attachment; individual wire versions can still be unsupported. Portable runtimes need standard JavaScript APIs including `AbortController`, `TextEncoder`, timers, maps, and sets. React is required only by your React consumer; compressed payload decoding is provided by the Node entry. + +The [SDK examples](examples) cover a bounded HTTP server, a frontend transport adapter, a reader-only consumer, React subscription, and an executable lifecycle walkthrough with receipts, retry, partial batches, cancellation, drain, JSON, and paging. Run `pnpm --filter @sentry/telemetry test` to typecheck the published declarations and every example, then execute the lifecycle walkthrough. CI runs this check after the package build; it does not add a whole-project TypeScript gate. + +### Getter reference + +| Read | Result | +| --------------------------------------------------------------- | --------------------------------------------------------------- | +| `getTrace(traceId)`, `getTraces(filter?)` | Assembled traces | +| `getEvent(eventId)`, `getEvents(filter?)`, `getErrors(filter?)` | Normalized events and captured exceptions | +| `getSpans(filter?)` | Normalized spans | +| `getLog(logId)`, `getLogs(filter?)` | Logs and trace correlation | +| `getDatabaseQuerySummaries(filter?)` | Database spans grouped by trimmed description | +| `getResourceSummaries(filter?)` | Resource timings and encoded sizes | +| `getProfile(traceId)`, `getProfileFunctionSummaries(filter?)` | Associated profiles and inclusive frame durations/sample counts | +| `getAITraces(filter?)` | AI interactions, tokens, prompts, responses, and tools | +| `getWebVitals(filter?)` | Scored page-load events | +| `getEnvelope(envelopeId)`, `getEnvelopes()` | Retained wire envelopes | +| `getSdks()` | Observed SDK identities | + +Event reads accept `traceId` and `spanId`; trace/log/span/profile/summary reads +accept `traceId`; query summaries also accept `description`. The time/order/limit +filters apply to filtered collection reads, not `getEnvelopes()` or `getSdks()`. +Summary fields name units explicitly, including `totalDurationMs`, +`averageDurationMs`, and `averageEncodedSizeBytes`. Resource summaries also expose +`averageSizeBytes` when a size is known: this averages finite, nonnegative captured +sizes using `http.response_content_length`, then `http.response.body.size`, then +`http.decoded_response_content_length` for each execution. Missing or invalid +sizes are excluded. `averageEncodedSizeBytes` retains its encoded-field-only +behavior and treats executions without a valid encoded size as zero. Stored envelopes retain wire +timestamp units. Caller input is copied; read results are readonly. + +`reset()` clears captured data and decoded history, keeps subscriptions, and +invalidates in-flight/queued work. `prune()` explicitly enforces retention. +`dispose()` closes the instance; later ingestion returns `ignored/disposed`. +Eviction rebuilds derived state from retained envelopes, with cost proportional +to retained telemetry. Choose smaller limits for large profiles or heavy streams. + +## Pure processing helpers + +The same portable entry also exports models that can be computed without retaining a telemetry session: + +```ts +import { + buildProfileModel, + groupWebVitalPages, + inspectAI, +} from "@sentry/telemetry"; + +const profile = buildProfileModel(rawProfilePayload, selectedThreadId); +// profile.root, profile.functions, profile.threads; durations are milliseconds. +const interaction = inspectAI(rawSpanPayload); +// Semantic fields, system instructions, messages, response, and tool calls. +const pages = groupWebVitalPages([ + { id: "page-load", transaction: "/checkout", measurements }, +]); +``` + +`buildProfileModel` supports profile and profile-chunk payloads, calculates sample intervals independently per thread, and exposes inclusive and self sample counts and durations. `inspectAI` returns `undefined` for unrelated payloads; `isAIInteraction` and `readSpanData` also expose the shared detection and legacy/typed-attribute handling. + +`groupWebVitalPages` groups visits by transaction and rates measurements using exported `WEB_VITALS` thresholds. It normalizes explicitly declared seconds to milliseconds, preserves CLS as a unitless value, and omits absent, negative, or non-finite measurements. These helpers return new caller-owned models and do not require React, a server, or a telemetry store. + +## Ownership and adoption + +This Apache-2.0 implementation originated in [Spotlight](https://github.com/getsentry/spotlight/pull/1374). Toolkit owns subsequent shared processing changes. The Local receiver imports `@sentry/telemetry/node`; the Local browser app imports `@sentry/telemetry`. UI rendering, HTTP/SSE transport, sessions, and terminal formatting stay in their applications. + +The package exports built JavaScript and declarations. Run `pnpm --filter @sentry/telemetry build` before direct source development in a consumer. Workspace build/test scripts prepare this dependency. The portable entry has no React or Node runtime dependency. + +After the first npm release, Spotlight can replace its extraction with imports from this package and remove its local processing implementation. Existing Spotlight `/sdk` transport helpers are a separate compatibility API. No npm publication occurs as part of the migration PR. + +## Release + +Use the repository Release workflow with product `telemetry`. The `packages/telemetry` Craft workspace publishes the independently versioned `@sentry/telemetry` npm artifact built by Post-merge tasks. The portable and Node consumer contracts and executable examples are checked against emitted declarations before packing. diff --git a/packages/telemetry/_fixtures/continuous_profiling/profile_chunk_1.txt b/packages/telemetry/_fixtures/continuous_profiling/profile_chunk_1.txt new file mode 100644 index 000000000..65cb89f48 --- /dev/null +++ b/packages/telemetry/_fixtures/continuous_profiling/profile_chunk_1.txt @@ -0,0 +1,3 @@ +{"event_id":"3e11a5c9831f4e49939c0a81944ea2cb","sent_at":"2024-08-27T15:00:11.503779Z","sdk":{"name":"sentry.cocoa","version":"8.36.0"}} +{"type":"profile_chunk","length":1643} +{"type":"profile_chunk","version":"2","profiler_id":"71bba98d90b545c39f2ae73f702d7ef4","chunk_id":"3e11a5c9831f4e49939c0a81944ea2cb","platform":"cocoa","release":"io.sentry.sample.iOS-Swift@8.36.0+1","environment":"simulator","client_sdk":{"name":"sentry.cocoa","version":"8.36.0"},"debug_meta":{"images":[{"debug_id":"5819FF25-01CB-3D32-B84F-0634B37D3BBC","image_addr":"0x00000001023a8000","type":"macho","image_size":16384,"code_file":"/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS17.2.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libLogRedirect.dylib"}]},"profile":{"samples":[{"timestamp":1724770811.503,"stack_id":0,"thread_id":"259"},{"timestamp":1724770811.513,"stack_id":1,"thread_id":"259"},{"timestamp":1724770811.523,"stack_id":2,"thread_id":"259"},{"timestamp":1724770811.533,"stack_id":3,"thread_id":"259"},{"timestamp":1724770811.543,"stack_id":2,"thread_id":"259"},{"timestamp":1724770811.553,"stack_id":1,"thread_id":"259"},{"timestamp":1724770811.563,"stack_id":0,"thread_id":"259"}],"stacks":[[0],[1,0],[2,1,0],[3,2,1,0]],"frames":[{"instruction_addr":"0x000000010232d144","function":"_main","filename":"main.m","lineno":14,"in_app":true},{"instruction_addr":"0x000000010232d200","function":"UIApplicationMain","module":"UIKit","in_app":false},{"instruction_addr":"0x000000010232d300","function":"-[AppDelegate application:didFinishLaunchingWithOptions:]","filename":"AppDelegate.swift","lineno":15,"in_app":true},{"instruction_addr":"0x000000010232d400","function":"-[AppDelegate setupSentry]","filename":"AppDelegate.swift","lineno":25,"in_app":true}],"thread_metadata":{"259":{"name":"main","priority":31}}}} diff --git a/packages/telemetry/_fixtures/continuous_profiling/transaction.txt b/packages/telemetry/_fixtures/continuous_profiling/transaction.txt new file mode 100644 index 000000000..eeb78ec6f --- /dev/null +++ b/packages/telemetry/_fixtures/continuous_profiling/transaction.txt @@ -0,0 +1,3 @@ +{"event_id":"txn_continuous_profile_123abc","sent_at":"2024-08-27T15:00:11.700000Z","trace":{"environment":"simulator","public_key":"abc123def456","trace_id":"f1e2d3c4b5a6978012345678901234ab"},"sdk":{"name":"sentry.cocoa","version":"8.36.0"}} +{"type":"transaction","length":2262} +{"type":"transaction","transaction":"iOS App Launch","transaction_info":{"source":"custom"},"contexts":{"trace":{"trace_id":"f1e2d3c4b5a6978012345678901234ab","span_id":"abc123def4567890","op":"app.launch","description":"iOS Application Launch","status":"ok","data":{"thread.id":"259","thread.name":"main"}},"profile":{"profiler_id":"71bba98d90b545c39f2ae73f702d7ef4"},"device":{"family":"iPhone","model":"iPhone14,2","arch":"arm64","simulator":true},"os":{"name":"iOS","version":"17.2"},"app":{"app_name":"iOS-Swift","app_version":"8.36.0","app_build":"1"}},"tags":{"environment":"simulator"},"timestamp":"2024-08-27T15:00:11.640000Z","start_timestamp":"2024-08-27T15:00:11.500000Z","spans":[{"trace_id":"f1e2d3c4b5a6978012345678901234ab","span_id":"span1234567890ab","parent_span_id":"abc123def4567890","op":"app.start.cold","description":"Cold App Start","start_timestamp":"2024-08-27T15:00:11.503779Z","timestamp":"2024-08-27T15:00:11.563779Z","status":"ok","data":{"thread.id":"259","thread.name":"main"}},{"trace_id":"f1e2d3c4b5a6978012345678901234ab","span_id":"span2345678901bc","parent_span_id":"abc123def4567890","op":"ui.load","description":"ViewController.viewDidLoad","start_timestamp":"2024-08-27T15:00:11.573779Z","timestamp":"2024-08-27T15:00:11.633779Z","status":"ok","data":{"thread.id":"259","thread.name":"main"}},{"trace_id":"f1e2d3c4b5a6978012345678901234ab","span_id":"span3456789012cd","parent_span_id":"span2345678901bc","op":"http.client","description":"GET /api/items","start_timestamp":"2024-08-27T15:00:11.583779Z","timestamp":"2024-08-27T15:00:11.623779Z","status":"ok","data":{"http.method":"GET","http.url":"https://api.example.com/api/items","http.status_code":200,"thread.id":"259"}}],"measurements":{"app_start_cold":{"value":140,"unit":"millisecond"},"frames_total":{"value":120,"unit":"none"},"frames_slow":{"value":2,"unit":"none"},"frames_frozen":{"value":0,"unit":"none"}},"event_id":"txn_continuous_profile_123abc","platform":"cocoa","release":"io.sentry.sample.iOS-Swift@8.36.0+1","environment":"simulator","sdk":{"name":"sentry.cocoa","version":"8.36.0","packages":[{"name":"cocoapods:sentry-cocoa","version":"8.36.0"}],"integrations":["AutoBreadcrumbTracking","AutoSessionTracking","Crash","Performance","Profiling"]}} diff --git a/packages/telemetry/_fixtures/envelope_span_v2.txt b/packages/telemetry/_fixtures/envelope_span_v2.txt new file mode 100644 index 000000000..e8e3482c5 --- /dev/null +++ b/packages/telemetry/_fixtures/envelope_span_v2.txt @@ -0,0 +1,3 @@ +{"sent_at":"2026-10-26T01:35:45.322Z","trace":{"environment":"production","public_key":"public","trace_id":"a9eab6940aba4ae7bf2f8ac5a54c6a2f","transaction":"GET /applications/[applicationId]/subscriptions","sampled":"true","sample_rand":"0.440015829087657","sample_rate":"1"},"sdk":{"name":"sentry.javascript.node","version":"11.9.0"}} +{"type":"span","item_count":2,"content_type":"application/vnd.sentry.items.span.v2+json"} +{"version":2,"items":[{"name":"SELECT \"public\".\"Customer\".\"id\" FROM \"public\".\"Customer\"","span_id":"832028f3635102b6","trace_id":"a9eab6940aba4ae7bf2f8ac5a54c6a2f","parent_span_id":"971f7be43c3ad277","start_timestamp":1793000000.125,"end_timestamp":1793000000.25,"is_segment":false,"status":"ok","attributes":{"sentry.origin":{"value":"manual","type":"string"},"sentry.op":{"value":"db","type":"string"},"db.system.name":{"value":"postgresql","type":"string"},"db.response.returned_rows":{"value":3,"type":"integer"},"sentry.trace_lifecycle":{"value":"stream","type":"string"},"sentry.segment.name":{"value":"GET /applications/[applicationId]/subscriptions","type":"string"},"sentry.segment.id":{"value":"971f7be43c3ad277","type":"string"},"sentry.sdk.name":{"value":"sentry.javascript.node","type":"string"},"sentry.sdk.version":{"value":"11.9.0","type":"string"},"sentry.environment":{"value":"production","type":"string"}}},{"name":"GET /applications/[applicationId]/subscriptions","span_id":"971f7be43c3ad277","trace_id":"a9eab6940aba4ae7bf2f8ac5a54c6a2f","start_timestamp":1793000000,"end_timestamp":1793000001,"is_segment":true,"status":"ok","attributes":{"sentry.origin":{"value":"manual","type":"string"},"sentry.op":{"value":"http.server","type":"string"},"sentry.segment.name.source":{"value":"custom","type":"string"},"sentry.sample_rate":{"value":1,"type":"integer"},"sentry.trace_lifecycle":{"value":"stream","type":"string"},"sentry.segment.name":{"value":"GET /applications/[applicationId]/subscriptions","type":"string"},"sentry.segment.id":{"value":"971f7be43c3ad277","type":"string"},"sentry.sdk.name":{"value":"sentry.javascript.node","type":"string"},"sentry.sdk.version":{"value":"11.9.0","type":"string"},"sentry.environment":{"value":"production","type":"string"},"sentry.sdk.integrations":{"value":["SpanStreaming"],"type":"array"}}}]} diff --git a/packages/telemetry/examples/frontend.ts b/packages/telemetry/examples/frontend.ts new file mode 100644 index 000000000..7fa946801 --- /dev/null +++ b/packages/telemetry/examples/frontend.ts @@ -0,0 +1,13 @@ +import { type IngestOptions, createTelemetryStore } from "@sentry/telemetry"; + +/** Keep one instance for the lifetime of the telemetry view. */ +export function createFrontendTelemetry() { + const telemetry = createTelemetryStore({ retention: { maxEnvelopes: 500 } }); + return { + telemetry, + // Pass the parsed envelopes received over SSE, WebSocket, or your own transport. + receiveEnvelope: (envelope: unknown, options?: IngestOptions) => + telemetry.ingestEnvelope(envelope, options), + disconnect: telemetry.dispose, + }; +} diff --git a/packages/telemetry/examples/lifecycle.ts b/packages/telemetry/examples/lifecycle.ts new file mode 100644 index 000000000..a00914ae6 --- /dev/null +++ b/packages/telemetry/examples/lifecycle.ts @@ -0,0 +1,124 @@ +import { SDK_CAPABILITIES, type SentryEnvelope } from "@sentry/telemetry"; +import { createTelemetryPipeline } from "@sentry/telemetry/node"; +import { createFrontendTelemetry } from "./frontend.js"; +import { inspectTrace, readEventIds } from "./reader.js"; + +// Parsed wire timestamps are seconds; normalized reader timestamps are Unix milliseconds. +function envelope(id: string): SentryEnvelope { + return [ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "example-trace", + span_id: id, + name: "SELECT 1", + start_timestamp: 1, + end_timestamp: 1.125, + attributes: { + "sentry.op": { type: "string", value: "db.query" }, + }, + }, + ], + }, + ], + ], + ] as const; +} +function check(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +const { telemetry, receiveEnvelope, disconnect } = createFrontendTelemetry(); +const pipeline = createTelemetryPipeline(); +try { + const input = envelope("one"); + const first = await receiveEnvelope(input, { deliveryId: "request-1" }); + check( + first.status === "processed" && first.retained && first.counts.spans === 1, + "Receipt counts", + ); + const retry = await receiveEnvelope(input, { deliveryId: "request-1" }); + check( + retry.status === "ignored" && retry.reason === "duplicate", + "Retained delivery retry", + ); + + // Each input gets a receipt. A malformed middle input does not roll back valid neighbors. + const batch = await telemetry.ingestEnvelopes([ + envelope("two"), + null, + envelope("three"), + ]); + check( + batch[0].status === "processed" && + batch[1].status === "invalid" && + batch[2].status === "processed", + "Partial batch", + ); + if (batch[1].status === "invalid") { + for (const diagnostic of batch[1].diagnostics) { + console.log(diagnostic.code, diagnostic.severity, diagnostic.itemIndex); + } + } + const controller = new AbortController(); + controller.abort(); + const cancelled = await telemetry.ingestEnvelope(envelope("cancelled"), { + signal: controller.signal, + }); + check( + cancelled.status === "ignored" && cancelled.reason === "aborted", + "Cancellation receipt", + ); + await telemetry.drain(); // A barrier for work already submitted, not future submissions. + check(telemetry.getStats().pending === 0, "Drained work"); + + const reader = inspectTrace(telemetry, "example-trace"); + check(JSON.parse(reader.snapshotJson).schemaVersion === 1, "JSON schema"); + check( + JSON.parse(reader.profileJson).schemaVersion === 1, + "Summary JSON schema", + ); + const page = telemetry.readPage("events", { limit: 1 }); + check(page.status === "ok" && page.nextCursor, "Bounded page"); + await receiveEnvelope(envelope("four")); + const stale = telemetry.readPage("events", { + limit: 1, + cursor: page.nextCursor, + }); + check( + stale.status === "invalid" && stale.code === "stale_cursor", + "Revision-bound cursor", + ); + check(readEventIds(telemetry).status === "ok", "Reader-only pagination"); + + const wire = (id: string) => + `{}\n${JSON.stringify({ type: "span" })}\n${JSON.stringify(envelope(id)[1][0][1])}`; + const payloads = await pipeline.ingestPayloads([ + { body: wire("node-one") }, + { body: "invalid" }, + ]); + check( + payloads[0].status === "processed" && payloads[1].status === "invalid", + "Payload batch", + ); + if (payloads[0].status === "processed") { + check( + pipeline.getDecodedPayload(payloads[0].envelopeId)?.body.length, + "Copied decoded payload", + ); + } + await pipeline.drain(); + console.log( + SDK_CAPABILITIES.timestampUnit, + first.counts, + telemetry.getStats(), + ); +} finally { + pipeline.dispose(); + disconnect(); +} diff --git a/packages/telemetry/examples/react.tsx b/packages/telemetry/examples/react.tsx new file mode 100644 index 000000000..31db31e92 --- /dev/null +++ b/packages/telemetry/examples/react.tsx @@ -0,0 +1,28 @@ +import type { TelemetryReader, TelemetryStore } from "@sentry/telemetry"; +import { useSyncExternalStore } from "react"; + +type Props = { + reader: TelemetryReader; + subscribe: TelemetryStore["subscribe"]; + traceId?: string; +}; + +/** Pass reader={telemetry} subscribe={telemetry.subscribe}; the view cannot ingest or reset. */ +export function QueryList({ reader, subscribe, traceId }: Props) { + useSyncExternalStore( + subscribe, + () => reader.getSnapshot(), + () => reader.getSnapshot(), + ); + const queries = reader.getDatabaseQuerySummaries({ traceId }); + return ( +
    + {queries.map((query) => ( +
  • + {query.description}: {query.count} calls, {query.totalDurationMs}ms + total, {query.averageDurationMs}ms average +
    • + ))} +
+ ); +} diff --git a/packages/telemetry/examples/reader.ts b/packages/telemetry/examples/reader.ts new file mode 100644 index 000000000..507d60c75 --- /dev/null +++ b/packages/telemetry/examples/reader.ts @@ -0,0 +1,40 @@ +import { + type TelemetryReader, + serializeSnapshot, + serializeSummaries, + serializeTrace, +} from "@sentry/telemetry"; + +/** Business logic only needs a reader, independent of how telemetry arrives. */ +export function inspectTrace(reader: TelemetryReader, traceId: string) { + const trace = reader.getTrace(traceId); + return { + queries: reader.getDatabaseQuerySummaries({ + traceId, + order: "desc", + limit: 10, + }), + detailedQueries: reader.getDatabaseQuerySummaries({ + traceId, + includeSpans: true, + }), + traceJson: trace ? JSON.stringify(serializeTrace(trace)) : undefined, + snapshotJson: JSON.stringify(serializeSnapshot(reader.getSnapshot())), + profileJson: JSON.stringify( + serializeSummaries(reader.getProfileFunctionSummaries({ traceId })), + ), + }; +} + +/** Keep the filters unchanged across pages; restart after any snapshot change. */ +export function readEventIds(reader: TelemetryReader) { + const ids: string[] = []; + let cursor: string | undefined; + do { + const page = reader.readPage("events", { limit: 1, order: "asc", cursor }); + if (page.status === "invalid") return page; // stale_cursor, invalid_cursor, or invalid_query + ids.push(...page.items.map((event) => event.event_id)); + cursor = page.nextCursor; + } while (cursor); + return { status: "ok" as const, ids }; +} diff --git a/packages/telemetry/examples/server.ts b/packages/telemetry/examples/server.ts new file mode 100644 index 000000000..0c0063402 --- /dev/null +++ b/packages/telemetry/examples/server.ts @@ -0,0 +1,79 @@ +import { createServer } from "node:http"; +import { createTelemetryPipeline } from "@sentry/telemetry/node"; + +const maxPayloadBytes = 8 * 1024 * 1024; + +/** Embed the SDK in an HTTP server; the SDK itself does not open a port. */ +export function createTelemetryServer() { + const pipeline = createTelemetryPipeline({ + maxPayloadBytes, + decodedHistory: { maxEntries: 100 }, + retention: { maxEnvelopes: 1000, maxAgeMs: 10 * 60 * 1000 }, + }); + const server = createServer(async (request, response) => { + try { + const url = new URL(request.url ?? "/", "http://localhost"); + response.setHeader("content-type", "application/json"); + if (request.method === "POST" && url.pathname === "/envelope") { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const buffer = Buffer.from(chunk); + bytes += buffer.length; + // Bound HTTP collection too: SDK limits only apply once the body is passed in. + if (bytes > maxPayloadBytes) { + response + .writeHead(413) + .end(JSON.stringify({ error: "Payload too large" })); + return; + } + chunks.push(buffer); + } + const controller = new AbortController(); + const abort = () => controller.abort(); + response.once("close", abort); + const deliveryId = request.headers["idempotency-key"]; + const result = await pipeline.ingestPayload({ + body: Buffer.concat(chunks), + contentType: request.headers["content-type"], + encoding: request.headers["content-encoding"], + userAgent: request.headers["user-agent"], + deliveryId: typeof deliveryId === "string" ? deliveryId : undefined, + signal: controller.signal, + }); + response.off("close", abort); + response.statusCode = + result.status === "invalid" + ? 400 + : result.status === "ignored" && + result.reason === "unsupported_content_type" + ? 415 + : result.status === "ignored" && + ["queue_full", "reset", "disposed", "aborted"].includes( + result.reason, + ) + ? 503 + : 202; + response.end(JSON.stringify(result)); + return; + } + if (request.method === "GET" && url.pathname === "/queries") { + const traceId = url.searchParams.get("traceId") ?? undefined; + response.end( + JSON.stringify(pipeline.getDatabaseQuerySummaries({ traceId })), + ); + return; + } + response.statusCode = 404; + response.end(JSON.stringify({ error: "Not found" })); + } catch { + // An aborted request body rejects its async iterator. Keep the server usable. + if (!response.destroyed) { + response.statusCode = 400; + response.end(JSON.stringify({ error: "Could not read request" })); + } + } + }); + server.on("close", pipeline.dispose); + return { server, pipeline }; +} diff --git a/packages/telemetry/examples/tsconfig.json b/packages/telemetry/examples/tsconfig.json new file mode 100644 index 000000000..b712e2234 --- /dev/null +++ b/packages/telemetry/examples/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023", "DOM"], + "types": ["node", "react"], + "jsx": "react-jsx", + "strict": true, + "skipLibCheck": false, + "rootDir": "." + }, + "include": ["*.ts", "*.tsx"] +} diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json new file mode 100644 index 000000000..ead247d89 --- /dev/null +++ b/packages/telemetry/package.json @@ -0,0 +1,55 @@ +{ + "name": "@sentry/telemetry", + "version": "0.1.0", + "description": "Shared Sentry envelope processing, trace assembly, and telemetry queries for browsers and servers", + "type": "module", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/getsentry/toolkit.git", + "directory": "packages/telemetry" + }, + "engines": { + "node": ">=20.19.0" + }, + "publishConfig": { + "access": "public" + }, + "sideEffects": false, + "files": [ + "dist/**/*.js", + "dist/**/*.d.ts", + "dist/**/*.js.map", + "LICENSE", + "README.md" + ], + "exports": { + ".": { + "types": "./dist/sdk/core.d.ts", + "import": "./dist/sdk/core.js" + }, + "./node": { + "types": "./dist/sdk/node.d.ts", + "import": "./dist/sdk/node.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "tsc": "tsc --noEmit", + "test": "pnpm run build && vitest run && node scripts/check-package.mjs", + "test:ci": "pnpm run build && vitest run --reporter=default --reporter=junit --outputFile=tests.junit.xml && node scripts/check-package.mjs", + "prepack": "pnpm run build" + }, + "dependencies": { + "@sentry/core": "^10.63.0", + "uuidv7": "^1.0.2", + "zustand": "^5.0.3" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:", + "react": "catalog:", + "@types/react": "catalog:" + } +} diff --git a/packages/telemetry/scripts/check-package.mjs b/packages/telemetry/scripts/check-package.mjs new file mode 100644 index 000000000..915c4678f --- /dev/null +++ b/packages/telemetry/scripts/check-package.mjs @@ -0,0 +1,153 @@ +import { spawnSync } from "node:child_process"; +import { + access, + cp, + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; + +const packageDirectory = fileURLToPath(new URL("../", import.meta.url)); +const require = createRequire(import.meta.url); +const tsc = require.resolve("typescript/bin/tsc"); +const tarball = process.argv[2] && resolve(process.argv[2]); +const temporary = await mkdtemp(join(tmpdir(), "telemetry-consumer-")); + +function command(binary, args, options = {}) { + const result = spawnSync(binary, args, { + cwd: packageDirectory, + stdio: "inherit", + ...options, + }); + if (result.error) throw result.error; + if (result.status !== 0) + throw new Error(`SDK check failed (${result.status ?? result.signal})`); + return result; +} + +try { + let consumerDirectory = packageDirectory; + if (tarball) { + const listing = command("tar", ["-tzf", tarball], { + encoding: "utf8", + stdio: "pipe", + }).stdout; + const entries = listing.trim().split(/\r?\n/); + for (const entry of entries) { + if ( + !entry.startsWith("package/") || + entry.includes("\\") || + entry.split("/").includes("..") + ) + throw new Error(`Unsafe packed path: ${entry}`); + if ( + entry.endsWith("/") && + (entry === "package/" || entry.startsWith("package/dist/")) + ) + continue; + if ( + /\.(?:test|spec)\./.test(entry) || + !/^package\/(?:package\.json|LICENSE|README\.md|dist\/.+\.(?:js|d\.ts|js\.map))$/.test( + entry, + ) + ) + throw new Error(`Unexpected packed file: ${entry}`); + } + const extracted = join(temporary, "unpacked"); + await mkdir(extracted); + command("tar", ["-xzf", tarball, "-C", extracted]); + const packedDirectory = join(extracted, "package"); + consumerDirectory = join(temporary, "consumer"); + await mkdir(consumerDirectory); + await cp( + join(packageDirectory, "tests", "types"), + join(consumerDirectory, "tests", "types"), + { recursive: true }, + ); + await cp( + join(packageDirectory, "examples"), + join(consumerDirectory, "examples"), + { recursive: true }, + ); + await writeFile( + join(consumerDirectory, "package.json"), + '{"type":"module"}\n', + ); + const scope = join(consumerDirectory, "node_modules", "@sentry"); + await mkdir(scope, { recursive: true }); + await symlink(packedDirectory, join(scope, "telemetry"), "junction"); + const manifest = JSON.parse( + await readFile(join(packedDirectory, "package.json"), "utf8"), + ); + // Only declared dependencies are made available to the extracted package. + for (const dependency of Object.keys(manifest.dependencies ?? {})) { + const target = join(packedDirectory, "node_modules", dependency); + await mkdir(dirname(target), { recursive: true }); + await symlink( + join(packageDirectory, "node_modules", dependency), + target, + "junction", + ); + } + // Node and React are development-only contracts for the consumer examples. + for (const dependency of ["@types/node", "@types/react", "react"]) { + const target = join(consumerDirectory, "node_modules", dependency); + await mkdir(dirname(target), { recursive: true }); + await symlink( + join(packageDirectory, "node_modules", dependency), + target, + "junction", + ); + } + for (const entry of ["core", "node"]) + for (const extension of ["js", "d.ts"]) + await access( + join(packedDirectory, "dist", "sdk", `${entry}.${extension}`), + ); + } + + // Resolve the public export map and emitted declarations, with no source aliases. + if (!tarball) { + for (const entry of ["core", "node"]) { + for (const extension of ["js", "d.ts"]) { + try { + await access( + join(packageDirectory, "dist", "sdk", `${entry}.${extension}`), + ); + } catch { + throw new Error( + "Build @sentry/telemetry before checking its public package", + ); + } + } + } + } + const run = (script, args = []) => + command(process.execPath, [script, ...args], { cwd: consumerDirectory }); + run(tsc, ["--project", "tests/types/tsconfig.sdk-core.json"]); + run(tsc, ["--project", "tests/types/tsconfig.sdk-node.json"]); + const output = join( + consumerDirectory === packageDirectory ? temporary : consumerDirectory, + "output", + ); + run(tsc, ["--project", "examples/tsconfig.json", "--outDir", output]); + await writeFile(join(output, "package.json"), '{"type":"module"}\n'); + if (!tarball) { + const scope = join(output, "node_modules", "@sentry"); + await mkdir(scope, { recursive: true }); + await symlink(packageDirectory, join(scope, "telemetry"), "junction"); + } + run(join(output, "lifecycle.js")); + console.log( + `${tarball ? "Packed" : "Built"} SDK declarations, consumer contracts, and executable examples passed.`, + ); +} finally { + await rm(temporary, { recursive: true, force: true }); +} diff --git a/packages/telemetry/src/sdk/core.ts b/packages/telemetry/src/sdk/core.ts new file mode 100644 index 000000000..d34f3d5ad --- /dev/null +++ b/packages/telemetry/src/sdk/core.ts @@ -0,0 +1,86 @@ +export { createTelemetryStore } from "../telemetry/index.ts"; +export type { + TelemetryStore, + TelemetryStoreOptions, + TelemetrySnapshot, + SentryEnvelope, + ProcessedEvent, + ProcessedSpan, + ProcessedTrace, + ProcessedLog, + ProcessedProfile, + AITrace, + QuerySummary, + DatabaseQueryFilter, + ResourceSummary, + ProfileFunctionSummary, + EventFilter, + TraceFilter, + RetentionOptions, + Diagnostic, + IngestResult, +} from "../telemetry/public.ts"; + +export { createTelemetryReader } from "../telemetry/reader.ts"; +export type { + TelemetryReader, + TelemetryReadFilter, + TraceReadFilter, + EventReadFilter, + SummaryReadFilter, + DatabaseReadFilter, + CompactQuerySummary, + CompactResourceSummary, + ReadCollection, + ReadCollectionItem, + ReadPageOptions, + ReadPageResult, +} from "../telemetry/reader.ts"; +export { + serializeTrace, + serializeSnapshot, + serializeSummaries, +} from "../telemetry/serialization.ts"; +export type { + JsonValue, + SerializedSpan, + SerializedTrace, + SerializedSnapshot, + SerializedSummaries, + SerializableSummary, +} from "../telemetry/serialization.ts"; +export { SDK_CAPABILITIES } from "../telemetry/capabilities.ts"; +export type { + DiagnosticCode, + IgnoreReason, + IngestCounts, + ReceiptMetadata, + IngestOptions, + BatchOptions, + ChangeMetadata, + IngestStats, +} from "../telemetry/contracts.ts"; + +export { buildProfileModel } from "../telemetry/profiles/model.ts"; +export type { + ProfileNode, + ProfileFunction, + ParsedProfile, +} from "../telemetry/profiles/model.ts"; +export { + inspectAI, + isAIInteraction, + readSpanData, +} from "../telemetry/ai/inspection.ts"; +export type { AIInspection } from "../telemetry/ai/inspection.ts"; +export { + groupWebVitalPages, + WEB_VITALS, +} from "../telemetry/selectors/vitalPages.ts"; +export type { + WebVitalEvent, + VitalName, + VitalMeasurement, + VitalVisit, + VitalPage, +} from "../telemetry/selectors/vitalPages.ts"; diff --git a/packages/telemetry/src/sdk/node.spec.ts b/packages/telemetry/src/sdk/node.spec.ts new file mode 100644 index 000000000..674b3318a --- /dev/null +++ b/packages/telemetry/src/sdk/node.spec.ts @@ -0,0 +1,1052 @@ +// @vitest-environment node +import { readFileSync } from "node:fs"; +import { runInNewContext } from "node:vm"; +import { brotliCompressSync, deflateSync, gzipSync } from "node:zlib"; +import { describe, expect, test, vi } from "vitest"; +import { createTelemetryPipeline } from "./node.ts"; + +const fixture = readFileSync( + new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url), +); + +describe("Node Telemetry adapter", () => { + test.each(["replay_recording", "future_binary"])( + "retains an opaque %s item alongside supported telemetry", + async (type) => { + const binary = deflateSync(Buffer.from("binary\nrecording")); + const body = Buffer.concat([ + Buffer.from( + '{}\n{"type":"event"}\n{"event_id":"with-opaque-item","timestamp":1}\n', + ), + Buffer.from(`${JSON.stringify({ type, length: binary.length })}\n`), + binary, + ]); + const telemetry = createTelemetryPipeline(); + const result = await telemetry.ingestPayload({ body }); + expect(result.status).toBe("processed"); + expect(telemetry.getEvent("with-opaque-item")).toBeDefined(); + expect(telemetry.getEnvelopes()[0][1][1][1]).toEqual({ + data: binary.toString("base64"), + }); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + expect(telemetry.getDecodedPayload(result.envelopeId)?.body).toEqual( + body, + ); + telemetry.dispose(); + }, + ); + + test.each([ + '{}\n{"type":"attachment","length":0}', + '{}\n{"type":"attachment"}', + ])( + "rejects a truncated item header without its terminating newline: %s", + async (body) => { + const telemetry = createTelemetryPipeline(); + expect((await telemetry.ingestPayload({ body })).status).toBe("invalid"); + expect(telemetry.getEnvelopes()).toEqual([]); + telemetry.dispose(); + }, + ); + + test("accepts an empty attachment payload after a complete item header", async () => { + const telemetry = createTelemetryPipeline(); + expect( + ( + await telemetry.ingestPayload({ + body: '{}\n{"type":"attachment","length":0}\n', + }) + ).status, + ).toBe("ignored"); + expect(telemetry.getEnvelopes()).toHaveLength(1); + telemetry.dispose(); + }); + + test.each([ + ["processed", fixture], + ["unsupported", Buffer.from('{}\n{"type":"attachment","length":5}\nhello')], + ] as const)( + "subscribers observe %s raw bytes together with the envelope in one notification", + async (_, body) => { + const spotlight = createTelemetryPipeline(); + const observations: (Buffer | undefined)[] = []; + spotlight.subscribe((snapshot) => { + const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); + observations.push(spotlight.getDecodedPayload(id)?.body); + }); + await spotlight.ingestPayload({ body }); + expect(observations).toEqual([body]); + }, + ); + + test("raw history bounds already apply when subscribers observe the next envelope", async () => { + const spotlight = createTelemetryPipeline({ + decodedHistory: { maxEntries: 1 }, + }); + const first = await spotlight.ingestPayload({ body: fixture }); + if (first.status !== "processed") + throw new Error("Expected processed envelope"); + const observations: { + old: Buffer | undefined; + newest: Buffer | undefined; + }[] = []; + spotlight.subscribe((snapshot) => { + const envelope = snapshot.envelopes.find( + (item) => String(item[0].__spotlight_envelope_id) !== first.envelopeId, + )!; + observations.push({ + old: spotlight.getDecodedPayload(first.envelopeId)?.body, + newest: spotlight.getDecodedPayload( + String(envelope[0].__spotlight_envelope_id), + )?.body, + }); + }); + await spotlight.ingestPayload({ body: fixture }); + expect(observations).toEqual([{ old: undefined, newest: fixture }]); + }); + + test("defaults to Sentry envelopes and exposes processed raw payloads", async () => { + const spotlight = createTelemetryPipeline(); + const result = await spotlight.ingestPayload({ body: fixture }); + expect(result.status).toBe("processed"); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([ + { count: 1, totalDurationMs: 125 }, + ]); + expect(spotlight.getDecodedPayload(result.envelopeId)).toEqual({ + body: fixture, + contentType: "application/x-sentry-envelope", + }); + }); + + test.each([ + ["identity", (body: Buffer) => body], + ["gzip", gzipSync], + ["deflate", deflateSync], + ["br", brotliCompressSync], + ] as const)("decodes %s payloads", async (encoding, compress) => { + const spotlight = createTelemetryPipeline(); + const result = await spotlight.ingestPayload({ + body: new Uint8Array(compress(fixture)), + contentType: " Application/X-Sentry-Envelope ; charset=utf-8 ", + encoding, + }); + expect(result.status).toBe("processed"); + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([ + { count: 1, totalDurationMs: 125 }, + ]); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual( + fixture, + ); + }); + + test("accepts serialized envelope strings", async () => { + const spotlight = createTelemetryPipeline(); + expect( + (await spotlight.ingestPayload({ body: fixture.toString("utf8") })) + .status, + ).toBe("processed"); + }); + + test("returns an explicit ignored result for unsupported content types", async () => { + const spotlight = createTelemetryPipeline(); + expect( + await spotlight.ingestPayload({ + body: fixture, + contentType: "application/json", + }), + ).toMatchObject({ + status: "ignored", + reason: expect.any(String), + }); + expect(spotlight.getEnvelopes()).toEqual([]); + }); + + test("retains raw payloads for valid envelopes containing only unsupported items", async () => { + const spotlight = createTelemetryPipeline(); + const body = Buffer.from('{}\n{"type":"attachment","length":5}\nhello'); + const result = await spotlight.ingestPayload({ body }); + expect(result).toMatchObject({ + status: "ignored", + reason: "unsupported_items", + envelopeId: expect.any(String), + }); + if (result.status !== "ignored" || !result.envelopeId) + throw new Error("Expected retained unsupported envelope"); + expect(spotlight.getEnvelopes()).toHaveLength(1); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual(body); + }); + + test.each([ + [{ body: fixture, encoding: "zstd" }, "unsupported_encoding"], + [{ body: fixture, encoding: "gzip" }, "decompression_failed"], + [{ body: "not an envelope" }, "invalid_envelope"], + [{ body: "null" }, "invalid_envelope"], + [{ body: "42" }, "invalid_envelope"], + [{ body: "[]" }, "invalid_envelope"], + [{ body: "{}\nnull\n{}" }, "invalid_envelope"], + [{ body: "{}\n42\n{}" }, "invalid_envelope"], + [{ body: "{}\n[]\n{}" }, "invalid_envelope"], + [{ body: '{}\n{"type":"event"}\nnot json' }, "invalid_envelope"], + [ + { body: '{}\n{"type":"attachment","length":100}\nshort' }, + "invalid_envelope", + ], + [ + { body: '{}\n{"type":"attachment","length":-1}\nshort' }, + "invalid_envelope", + ], + [ + { body: '{}\n{"type":"attachment","length":1.5}\nshort' }, + "invalid_envelope", + ], + [ + { body: '{}\n{"type":"attachment","length":"5"}\nshort' }, + "invalid_envelope", + ], + [ + { body: '{}\n{"type":"attachment","length":3}\nshort' }, + "invalid_envelope", + ], + [{ body: 42 }, "invalid_body"], + ])( + "returns diagnostics for malformed input %# without logging", + async (input, code) => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const warnings = vi.spyOn(console, "warn").mockImplementation(() => {}); + const logs = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + const spotlight = createTelemetryPipeline(); + // Runtime validation is part of the public boundary. + const result = await spotlight.ingestPayload( + input as Parameters[0], + ); + expect(result).toMatchObject({ + status: "invalid", + diagnostics: [{ code }], + }); + expect(spotlight.getEnvelopes()).toEqual([]); + expect(errors).not.toHaveBeenCalled(); + expect(warnings).not.toHaveBeenCalled(); + expect(logs).not.toHaveBeenCalled(); + } finally { + vi.restoreAllMocks(); + } + }, + ); + + test("copies incoming and returned raw payload bytes", async () => { + const spotlight = createTelemetryPipeline(); + const body = Buffer.from(fixture); + const pending = spotlight.ingestPayload({ body }); + body.fill(0); + const result = await pending; + expect(result.status).toBe("processed"); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + const raw = spotlight.getDecodedPayload(result.envelopeId)!; + expect(raw.body).toEqual(fixture); + raw.body.fill(0); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual( + fixture, + ); + }); + + test("bounds raw payload history independently from processed retention", async () => { + const spotlight = createTelemetryPipeline({ + decodedHistory: { maxEntries: 1 }, + }); + const first = await spotlight.ingestPayload({ body: fixture }); + const second = await spotlight.ingestPayload({ body: fixture }); + if (first.status !== "processed" || second.status !== "processed") + throw new Error("Expected envelopes"); + expect(spotlight.getDecodedPayload(first.envelopeId)).toBeUndefined(); + expect(spotlight.getDecodedPayload(second.envelopeId)?.body).toEqual( + fixture, + ); + expect(spotlight.getEnvelopes()).toHaveLength(2); + spotlight.reset(); + expect(spotlight.getDecodedPayload(second.envelopeId)).toBeUndefined(); + expect(spotlight.getEnvelopes()).toEqual([]); + }); + + test("invalid ingestion cannot evict a retained raw payload", async () => { + const spotlight = createTelemetryPipeline({ + decodedHistory: { maxEntries: 1 }, + }); + const first = await spotlight.ingestPayload({ body: fixture }); + if (first.status !== "processed") + throw new Error("Expected processed envelope"); + expect( + (await spotlight.ingestPayload({ body: "not an envelope" })).status, + ).toBe("invalid"); + expect((await spotlight.ingestPayload({ body: "{}\n{}\n{}" })).status).toBe( + "invalid", + ); + expect(spotlight.getDecodedPayload(first.envelopeId)?.body).toEqual( + fixture, + ); + expect(spotlight).not.toHaveProperty("buffer"); + expect(spotlight).not.toHaveProperty("store"); + }); + + test("retains at most 500 raw payloads by default", async () => { + const spotlight = createTelemetryPipeline({ + retention: { maxEnvelopes: 1 }, + }); + const first = await spotlight.ingestPayload({ body: fixture }); + let last = first; + for (let i = 0; i < 500; i++) + last = await spotlight.ingestPayload({ body: fixture }); + if (first.status !== "processed" || last.status !== "processed") + throw new Error("Expected envelopes"); + expect(spotlight.getDecodedPayload(first.envelopeId)).toBeUndefined(); + expect(spotlight.getDecodedPayload(last.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getEnvelopes()).toHaveLength(1); + }); + + test("zero raw buffer capacity still processes telemetry", async () => { + const spotlight = createTelemetryPipeline({ + decodedHistory: { maxEntries: 0 }, + }); + const result = await spotlight.ingestPayload({ body: fixture }); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([ + { count: 1, totalDurationMs: 125 }, + ]); + }); + + test("processed age retention does not expire the separate raw history", async () => { + vi.useFakeTimers(); + const spotlight = createTelemetryPipeline({ retention: { maxAgeMs: 100 } }); + try { + const result = await spotlight.ingestPayload({ body: fixture }); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + await vi.advanceTimersByTimeAsync(101); + expect(spotlight.getEnvelopes()).toEqual([]); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual( + fixture, + ); + } finally { + spotlight.dispose(); + vi.useRealTimers(); + } + }); + + test.each(["reset", "dispose"] as const)( + "%s during enrichment cannot restore raw history", + async (action) => { + let start!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + start = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const spotlight = createTelemetryPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const pending = spotlight.ingestPayload({ + body: [ + "{}", + '{"type":"event"}', + JSON.stringify({ + event_id: "pending-error", + timestamp: 1, + exception: { values: [{ type: "Error", value: "broken" }] }, + }), + ].join("\n"), + }); + await started; + spotlight[action](); + release(); + const result = await pending; + expect(result.status).toBe("ignored"); + if ("envelopeId" in result && result.envelopeId) + expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); + expect(spotlight.getEnvelopes()).toEqual([]); + }, + ); + + test("delivers invalid-input diagnostics to the configured observer", async () => { + const diagnostics: string[] = []; + const spotlight = createTelemetryPipeline({ + onDiagnostic: (diagnostic) => { + diagnostics.push(diagnostic.code); + }, + }); + const result = await spotlight.ingestPayload({ + body: fixture, + encoding: "zstd", + }); + expect(result.status).toBe("invalid"); + expect(diagnostics).toEqual(["unsupported_encoding"]); + }); + + test("throwing diagnostic observers do not reject invalid input", async () => { + const spotlight = createTelemetryPipeline({ + onDiagnostic: () => { + throw new Error("observer failed"); + }, + }); + expect( + (await spotlight.ingestPayload({ body: fixture, encoding: "zstd" })) + .status, + ).toBe("invalid"); + }); + + test("reset from a subscriber cannot repopulate raw payload history", async () => { + const spotlight = createTelemetryPipeline(); + let reset = false; + spotlight.subscribe(() => { + if (!reset && spotlight.getEnvelopes().length > 0) { + reset = true; + spotlight.reset(); + } + }); + const result = await spotlight.ingestPayload({ body: fixture }); + expect(reset).toBe(true); + if ("envelopeId" in result && result.envelopeId) + expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); + expect(spotlight.getEnvelopes()).toEqual([]); + }); + + test("dispose clears raw payloads and prevents subsequent ingestion", async () => { + const spotlight = createTelemetryPipeline(); + const result = await spotlight.ingestPayload({ body: fixture }); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + spotlight.dispose(); + expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); + expect((await spotlight.ingestPayload({ body: fixture })).status).toBe( + "ignored", + ); + expect(spotlight.getEnvelopes()).toEqual([]); + }); +}); + +describe("bounded payload ingestion", () => { + test.each([ + ["gzip", gzipSync], + ["deflate", deflateSync], + ["br", brotliCompressSync], + ] as const)( + "rejects %s expansion at the decompressor output limit", + async (encoding, compress) => { + const pipeline = createTelemetryPipeline({ maxDecodedBytes: 128 }); + const result = await pipeline.ingestPayload({ + body: compress(Buffer.alloc(4096, 65)), + encoding, + }); + expect(result).toMatchObject({ + status: "invalid", + retained: false, + diagnostics: [{ code: "decoded_payload_too_large" }], + }); + expect(pipeline.getEnvelopes()).toEqual([]); + pipeline.dispose(); + }, + ); + + test("enforces input and uncompressed decoded byte limits", async () => { + const inputLimited = createTelemetryPipeline({ maxPayloadBytes: 10 }); + expect(await inputLimited.ingestPayload({ body: fixture })).toMatchObject({ + status: "invalid", + diagnostics: [{ code: "payload_too_large" }], + }); + inputLimited.dispose(); + const decodedLimited = createTelemetryPipeline({ maxDecodedBytes: 10 }); + expect(await decodedLimited.ingestPayload({ body: fixture })).toMatchObject( + { + status: "invalid", + diagnostics: [{ code: "decoded_payload_too_large" }], + }, + ); + decodedLimited.dispose(); + }); + + test("evicts raw bytes by total decoded size before notifying subscribers", async () => { + const pipeline = createTelemetryPipeline({ + decodedHistory: { maxBytes: fixture.length }, + }); + const first = await pipeline.ingestPayload({ body: fixture }); + if (first.status !== "processed") + throw new Error("Expected processed envelope"); + const observations: unknown[] = []; + pipeline.subscribe((snapshot) => { + const newest = String( + snapshot.envelopes.at(-1)![0].__spotlight_envelope_id, + ); + observations.push([ + pipeline.getDecodedPayload(first.envelopeId), + pipeline.getDecodedPayload(newest)?.body, + ]); + }); + const second = await pipeline.ingestPayload({ body: fixture }); + expect(second.status).toBe("processed"); + expect(observations).toEqual([[undefined, fixture]]); + pipeline.dispose(); + }); + + test("zero raw byte budget disables decoded history", async () => { + const pipeline = createTelemetryPipeline({ + decodedHistory: { maxBytes: 0 }, + }); + const result = await pipeline.ingestPayload({ body: fixture }); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + expect(pipeline.getDecodedPayload(result.envelopeId)).toBeUndefined(); + pipeline.dispose(); + }); + + test("forwards delivery identity and abort signals to processing", async () => { + const pipeline = createTelemetryPipeline(); + const first = await pipeline.ingestPayload({ + body: fixture, + deliveryId: "wire-delivery", + }); + const duplicate = await pipeline.ingestPayload({ + body: fixture, + deliveryId: "wire-delivery", + }); + expect(first.status).toBe("processed"); + expect(duplicate).toMatchObject({ status: "ignored", reason: "duplicate" }); + const controller = new AbortController(); + controller.abort(); + expect( + await pipeline.ingestPayload({ + body: fixture, + signal: controller.signal, + }), + ).toMatchObject({ + status: "ignored", + reason: "aborted", + }); + expect(pipeline.getEnvelopes()).toHaveLength(1); + pipeline.dispose(); + }); + + test.each(["reset", "dispose"] as const)( + "%s cancels asynchronous decompression", + async (action) => { + const pipeline = createTelemetryPipeline(); + const pending = pipeline.ingestPayload({ + body: gzipSync(fixture), + encoding: "gzip", + }); + await Promise.resolve(); + pipeline[action](); + expect(await pending).toMatchObject({ + status: "ignored", + reason: action === "dispose" ? "disposed" : "reset", + }); + expect(pipeline.getEnvelopes()).toEqual([]); + pipeline.dispose(); + }, + ); + + test("bounds concurrently admitted payloads and drain includes decoding", async () => { + const pipeline = createTelemetryPipeline({ maxPending: 1 }); + const first = pipeline.ingestPayload({ + body: gzipSync(fixture), + encoding: "gzip", + }); + expect(await pipeline.ingestPayload({ body: fixture })).toMatchObject({ + status: "ignored", + reason: "queue_full", + }); + await pipeline.drain(); + expect((await first).status).toBe("processed"); + expect(pipeline.getEnvelopes()).toHaveLength(1); + pipeline.dispose(); + }); + + test("payload batches retain order and isolate invalid entries", async () => { + const pipeline = createTelemetryPipeline(); + const results = await pipeline.ingestPayloads([ + { body: fixture, deliveryId: "first" }, + { body: "invalid" }, + { body: fixture, deliveryId: "first" }, + ]); + expect(results.map((result) => result.status)).toEqual([ + "processed", + "invalid", + "ignored", + ]); + expect(results[2]).toMatchObject({ reason: "duplicate" }); + pipeline.dispose(); + }); +}); + +describe("payload lifecycle boundaries", () => { + test("delivery IDs key decoded history before public subscriptions", async () => { + const pipeline = createTelemetryPipeline(); + const observed: unknown[] = []; + pipeline.subscribe((snapshot) => { + const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); + observed.push([id, pipeline.getDecodedPayload(id)?.body]); + }); + const result = await pipeline.ingestPayload({ + body: fixture, + deliveryId: "effective-wire-id", + }); + expect(result).toMatchObject({ + status: "processed", + envelopeId: "effective-wire-id", + }); + expect(observed).toEqual([["effective-wire-id", fixture]]); + pipeline.dispose(); + }); + + test("an abort during enrichment releases the payload queue without publishing raw data", async () => { + let start!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + start = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const controller = new AbortController(); + const pipeline = createTelemetryPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const pending = pipeline.ingestPayload({ + body: '{}\n{"type":"event"}\n{"event_id":"aborted-error","timestamp":1,"exception":{"values":[]}}', + signal: controller.signal, + deliveryId: "aborted-wire-id", + }); + await started; + controller.abort(); + expect(await pending).toMatchObject({ + status: "ignored", + reason: "aborted", + }); + expect(pipeline.getDecodedPayload("aborted-wire-id")).toBeUndefined(); + expect((await pipeline.ingestPayload({ body: fixture })).status).toBe( + "processed", + ); + release(); + await pipeline.drain(); + expect(pipeline.getEvent("aborted-error")).toBeUndefined(); + pipeline.dispose(); + }); + + test("aborting decompression does not retain or process the decoded output", async () => { + const pipeline = createTelemetryPipeline(); + const controller = new AbortController(); + const pending = pipeline.ingestPayload({ + body: gzipSync(fixture), + encoding: "gzip", + signal: controller.signal, + }); + await Promise.resolve(); + controller.abort(); + expect(await pending).toMatchObject({ + status: "ignored", + reason: "aborted", + }); + expect(pipeline.getEnvelopes()).toEqual([]); + pipeline.dispose(); + }); + + test.each([ + "maxPayloadBytes", + "maxDecodedBytes", + "maxBytes", + "maxEntries", + ] as const)("validates %s configuration", (name) => { + for (const invalid of [ + -1, + 0.1, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + ]) { + const options = + name === "maxBytes" || name === "maxEntries" + ? { decodedHistory: { [name]: invalid } } + : { [name]: invalid }; + expect(() => createTelemetryPipeline(options)).toThrow(RangeError); + } + if (name === "maxPayloadBytes" || name === "maxDecodedBytes") { + expect(() => createTelemetryPipeline({ [name]: 0 })).toThrow(RangeError); + } + }); +}); + +test("Node pending stats include queued decode work and reset releases admission capacity", async () => { + const pipeline = createTelemetryPipeline({ maxPending: 1 }); + const old = pipeline.ingestPayload({ + body: gzipSync(fixture), + encoding: "gzip", + }); + expect(pipeline.getStats().pending).toBe(1); + await Promise.resolve(); + pipeline.reset(); + expect(pipeline.getStats().pending).toBe(0); + const next = pipeline.ingestPayload({ body: fixture }); + expect(pipeline.getStats().pending).toBe(1); + expect(await old).toMatchObject({ status: "ignored", reason: "reset" }); + expect((await next).status).toBe("processed"); + expect(pipeline.getStats().pending).toBe(0); + pipeline.dispose(); +}); + +test("Node drain excludes payloads admitted after its barrier", async () => { + let start!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + start = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const pipeline = createTelemetryPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const first = pipeline.ingestPayload({ body: fixture }); + const drained = pipeline.drain(); + const later = pipeline.ingestPayload({ + body: '{}\n{"type":"event"}\n{"event_id":"later-error","timestamp":1,"exception":{"values":[]}}', + }); + await started; + await drained; + expect((await first).status).toBe("processed"); + expect(pipeline.getEvent("later-error")).toBeUndefined(); + release(); + expect((await later).status).toBe("processed"); + pipeline.dispose(); +}); + +test("drain includes every entry of an already-submitted payload batch", async () => { + const pipeline = createTelemetryPipeline({ maxPending: 1 }); + const batch = pipeline.ingestPayloads([ + { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, + { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-second" }, + ]); + await pipeline.drain(); + expect(pipeline.getEnvelopes()).toHaveLength(2); + expect((await batch).map((result) => result.status)).toEqual([ + "processed", + "processed", + ]); + pipeline.dispose(); +}); + +test("reset cancels the remaining entries of a submitted payload batch", async () => { + const pipeline = createTelemetryPipeline(); + const batch = pipeline.ingestPayloads([ + { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, + { body: fixture, deliveryId: "batch-second" }, + ]); + await Promise.resolve(); + pipeline.reset(); + const results = await batch; + expect(results).toMatchObject([ + { status: "ignored", reason: "reset" }, + { status: "ignored", reason: "reset" }, + ]); + expect(pipeline.getEnvelopes()).toEqual([]); + expect(pipeline.getDecodedPayload("batch-second")).toBeUndefined(); + pipeline.dispose(); +}); + +test("a queued payload batch stays ahead of later payload work", async () => { + let start!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + start = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const pipeline = createTelemetryPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const batch = pipeline.ingestPayloads([ + { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, + { body: fixture, deliveryId: "batch-second" }, + ]); + const drained = pipeline.drain(); + const later = pipeline.ingestPayload({ + body: '{}\n{"type":"event"}\n{"event_id":"later-blocker","timestamp":1,"exception":{"values":[]}}', + }); + await started; + expect(pipeline.getEnvelope("batch-second")).toBeDefined(); + await drained; + expect((await batch).map((result) => result.status)).toEqual([ + "processed", + "processed", + ]); + release(); + await later; + pipeline.dispose(); +}); + +describe("one Node admission queue for every input method", () => { + const parsedEvent = (id: string, error = false) => [ + {}, + [ + [ + { type: "event" }, + { + event_id: id, + timestamp: 1, + ...(error ? { exception: { values: [] } } : {}), + }, + ], + ], + ]; + + test.each(["ingestEnvelope", "ingestEnvelopes"] as const)( + "wire admission bounds later %s calls", + async (method) => { + const pipeline = createTelemetryPipeline({ maxPending: 1 }); + const wire = pipeline.ingestPayload({ + body: gzipSync(fixture), + encoding: "gzip", + }); + const input = parsedEvent("later-parsed"); + const later = + method === "ingestEnvelope" + ? await pipeline.ingestEnvelope(input) + : (await pipeline.ingestEnvelopes([input]))[0]; + expect(later).toMatchObject({ status: "ignored", reason: "queue_full" }); + expect(pipeline.getStats().pending).toBe(1); + expect((await wire).status).toBe("processed"); + expect(pipeline.getEvent("later-parsed")).toBeUndefined(); + pipeline.dispose(); + }, + ); + + test.each(["ingestPayload", "ingestPayloads"] as const)( + "parsed admission bounds later %s calls", + async (method) => { + const pipeline = createTelemetryPipeline({ maxPending: 1 }); + const parsed = pipeline.ingestEnvelope(parsedEvent("first-parsed")); + const input = { body: fixture }; + const later = + method === "ingestPayload" + ? await pipeline.ingestPayload(input) + : (await pipeline.ingestPayloads([input]))[0]; + expect(later).toMatchObject({ status: "ignored", reason: "queue_full" }); + expect(pipeline.getStats().pending).toBe(1); + expect((await parsed).status).toBe("processed"); + pipeline.dispose(); + }, + ); + + test.each(["ingestEnvelope", "ingestEnvelopes"] as const)( + "wire drain excludes a later gated %s", + async (method) => { + let start!: () => void; + let release!: () => void; + const started = new Promise((resolve) => { + start = resolve; + }); + const gate = new Promise((resolve) => { + release = resolve; + }); + const pipeline = createTelemetryPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const wire = pipeline.ingestPayload({ + body: gzipSync(fixture), + encoding: "gzip", + }); + const drained = pipeline.drain(); + const input = parsedEvent("later-gated-parsed", true); + const later = + method === "ingestEnvelope" + ? pipeline.ingestEnvelope(input) + : pipeline.ingestEnvelopes([input]); + await started; + // The wire job must commit first; its already-captured barrier must settle while later work is gated. + expect(pipeline.getDatabaseQuerySummaries()).toHaveLength(1); + await drained; + expect((await wire).status).toBe("processed"); + release(); + await later; + pipeline.dispose(); + }, + ); + + test("parsed batches retain order ahead of later wire jobs", async () => { + const pipeline = createTelemetryPipeline(); + const seen: string[] = []; + pipeline.subscribe((_snapshot, change) => { + if (change.type === "ingest") seen.push(...change.envelopeIds); + }); + const batch = pipeline.ingestEnvelopes([ + [{ __spotlight_envelope_id: "parsed-first" }, []], + [{ __spotlight_envelope_id: "parsed-second" }, []], + ]); + const wire = pipeline.ingestPayload({ + body: fixture, + deliveryId: "wire-third", + }); + await pipeline.drain(); + expect(seen).toEqual(["parsed-first", "parsed-second", "wire-third"]); + expect((await batch).map((item) => item.status)).toEqual([ + "ignored", + "ignored", + ]); + expect((await wire).status).toBe("processed"); + pipeline.dispose(); + }); + + test("copied standalone parsed inputs retain weak object identity across calls and batches", async () => { + const pipeline = createTelemetryPipeline(); + const input = parsedEvent("copied-input"); + const pending = pipeline.ingestEnvelope(input); + input[1] = []; + const first = await pending; + expect(first.status).toBe("processed"); + expect(pipeline.getEvent("copied-input")).toBeDefined(); + const second = await pipeline.ingestEnvelope(input); + expect(second).toMatchObject({ + status: "ignored", + reason: "duplicate", + envelopeId: "envelopeId" in first ? first.envelopeId : "missing", + }); + expect( + (await pipeline.ingestEnvelopes([input, input])).map( + (item) => item.status, + ), + ).toEqual(["ignored", "ignored"]); + expect(pipeline.getEnvelopes()).toHaveLength(1); + expect(input[0]).not.toHaveProperty("__spotlight_envelope_id"); + pipeline.dispose(); + }); + + test("prune calls coalesce without exceeding a full shared queue", async () => { + const pipeline = createTelemetryPipeline({ maxPending: 1 }); + const wire = pipeline.ingestPayload({ + body: gzipSync(fixture), + encoding: "gzip", + }); + const first = pipeline.prune(); + const second = pipeline.prune(); + expect(second).toBe(first); + expect(pipeline.getStats().pending).toBe(1); + await Promise.all([wire, first, pipeline.drain()]); + expect(pipeline.getStats().pending).toBe(0); + pipeline.dispose(); + }); +}); + +test("parsed batch clone failures produce per-item receipts and preserve later inputs", async () => { + const pipeline = createTelemetryPipeline(); + const uncloneable = [{ bad: () => {} }, []]; + const good = [ + {}, + [[{ type: "event" }, { event_id: "after-clone-failure", timestamp: 1 }]], + ]; + const results = await pipeline.ingestEnvelopes([uncloneable, good, good]); + expect(results).toMatchObject([ + { status: "invalid", diagnostics: [{ code: "invalid_envelope" }] }, + { status: "processed" }, + { status: "ignored", reason: "duplicate" }, + ]); + expect(pipeline.getEnvelopes()).toHaveLength(1); + expect(Object.isFrozen(results[0].diagnostics[0])).toBe(true); + pipeline.dispose(); +}); + +test("reset invalidates queued parsed jobs and the remaining parsed batch entries", async () => { + const pipeline = createTelemetryPipeline(); + const wire = pipeline.ingestPayload({ + body: gzipSync(fixture), + encoding: "gzip", + }); + const parsed = pipeline.ingestEnvelope([{}, []]); + const batch = pipeline.ingestEnvelopes([ + [{}, []], + [{}, []], + ]); + pipeline.reset(); + expect(await wire).toMatchObject({ status: "ignored", reason: "reset" }); + expect(await parsed).toMatchObject({ status: "ignored", reason: "reset" }); + expect(await batch).toMatchObject([ + { status: "ignored", reason: "reset" }, + { status: "ignored", reason: "reset" }, + ]); + expect(pipeline.getStats().pending).toBe(0); + expect(pipeline.getEnvelopes()).toEqual([]); + pipeline.dispose(); +}); + +test("diagnostic callbacks cannot bypass shared admission during core validation", async () => { + const later: Promise[] = []; + const observed: number[] = []; + const pipeline = createTelemetryPipeline({ + maxPending: 1, + onDiagnostic() { + observed.push(pipeline.getStats().pending); + later.push(pipeline.ingestPayload({ body: fixture })); + }, + }); + const invalid = await pipeline.ingestEnvelope([ + {}, + [[{ type: "span" }, { span_id: "invalid" }]], + ]); + expect(invalid.status).toBe("invalid"); + expect(observed).toEqual([1]); + expect(await Promise.all(later)).toMatchObject([ + { status: "ignored", reason: "queue_full" }, + ]); + pipeline.dispose(); +}); + +test("accepts Uint8Array bodies created in another realm", async () => { + const body: Uint8Array = runInNewContext("Uint8Array.from(bytes)", { + bytes: [...fixture], + }); + expect(body instanceof Uint8Array).toBe(false); + const telemetry = createTelemetryPipeline(); + try { + expect((await telemetry.ingestPayload({ body })).status).toBe("processed"); + expect(telemetry.getTraces().length).toBeGreaterThan(0); + } finally { + telemetry.dispose(); + } +}); + +test("generated payload identities are canonical UUID strings", async () => { + const telemetry = createTelemetryPipeline(); + try { + const result = await telemetry.ingestPayload({ body: fixture }); + expect(result.status).toBe("processed"); + if (result.status !== "processed") + throw new Error("Expected processed envelope"); + expect(result.envelopeId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + } finally { + telemetry.dispose(); + } +}); diff --git a/packages/telemetry/src/sdk/node.ts b/packages/telemetry/src/sdk/node.ts new file mode 100644 index 000000000..cd04bdbbf --- /dev/null +++ b/packages/telemetry/src/sdk/node.ts @@ -0,0 +1,645 @@ +import { stringifyValue } from "../telemetry/utils/stringifyValue.ts"; +import { Buffer } from "node:buffer"; +import { types as nodeTypes } from "node:util"; +import { brotliDecompress, gunzip, inflate } from "node:zlib"; +import type { Envelope } from "@sentry/core"; +import type { + BatchOptions, + DiagnosticCode, + IgnoreReason, + IngestOptions, +} from "../telemetry/contracts.ts"; +import { decodeEnvelope } from "../telemetry/envelope/codec.ts"; +import { + normalizeEnvelope, + validateEnvelope, +} from "../telemetry/internal/envelopes.ts"; +import { + diagnostic, + receipt, + reportDiagnostic, +} from "../telemetry/internal/results.ts"; +import { validateRetention } from "../telemetry/internal/retention.ts"; +import { generateUuidv4 } from "../telemetry/uuid.ts"; +import { + type IngestResult, + type TelemetryStore, + type TelemetryStoreOptions, + createTelemetryStore, +} from "./core.ts"; + +export type { TelemetrySnapshot } from "./core.ts"; + +export type TelemetryPipelineOptions = TelemetryStoreOptions & { + decodedHistory?: { + /** Maximum retained decoded payloads (default 500); zero disables storage. */ + maxEntries?: number; + /** Total decoded payload history budget (default 32 MiB); zero disables storage. */ + maxBytes?: number; + }; + /** Maximum input body size before decompression. Default: 8 MiB. */ + maxPayloadBytes?: number; + /** Maximum body size after decompression. Default: 32 MiB. */ + maxDecodedBytes?: number; +}; + +export type TelemetryPayload = { + body: Uint8Array | string; + contentType?: string; + encoding?: string; + userAgent?: string; + deliveryId?: string; + signal?: AbortSignal; +}; + +export type DecodedPayload = { body: Buffer; contentType: string }; +export type TelemetryPipeline = TelemetryStore & { + ingestPayload(input: TelemetryPayload): Promise; + ingestPayloads( + inputs: readonly TelemetryPayload[], + ): Promise; + getDecodedPayload(envelopeId: string): DecodedPayload | undefined; +}; + +const envelopeContentType = "application/x-sentry-envelope"; +const compressedEncodings = new Set(["gzip", "deflate", "br"]); + +function limit(name: string, value: number, allowZero = false): number { + if (!Number.isSafeInteger(value) || value < (allowZero ? 0 : 1)) { + throw new RangeError( + `${name} must be a ${allowZero ? "non-negative" : "positive"} safe integer`, + ); + } + return value; +} + +/** Decode bounded serialized Sentry envelopes, then process them through the shared telemetry store. */ +export function createTelemetryPipeline( + options: TelemetryPipelineOptions = {}, +): TelemetryPipeline { + const maxHistoryEntries = limit( + "decodedHistory.maxEntries", + options.decodedHistory?.maxEntries ?? 500, + true, + ); + const maxHistoryBytes = limit( + "decodedHistory.maxBytes", + options.decodedHistory?.maxBytes ?? 32 * 1024 * 1024, + true, + ); + const maxPayloadBytes = limit( + "maxPayloadBytes", + options.maxPayloadBytes ?? 8 * 1024 * 1024, + ); + const maxDecodedBytes = limit( + "maxDecodedBytes", + options.maxDecodedBytes ?? 32 * 1024 * 1024, + ); + const maxPending = limit("maxPending", options.maxPending ?? 64); + const maxEnvelopeBytes = options.maxEnvelopeBytes ?? 32 * 1024 * 1024; + const telemetry = createTelemetryStore(options); + const decodedPayloads = new Map(); + const stagedPayloads = new Map(); + const pending = new Set>(); + const decodeControllers = new Set(); + const coreAdmissions = new Set(); + const identities = new WeakMap(); + let decodedBytes = 0; + let generation = 0; + let disposed = false; + let queue = Promise.resolve(); + let pruning: Promise | undefined; + + function retainDecodedPayload(envelopeId: string, payload: DecodedPayload) { + // Oversized individual payloads are processed but cannot enter the decoded history. + if ( + maxHistoryEntries === 0 || + payload.body.length > maxHistoryBytes || + maxHistoryBytes === 0 + ) + return; + const previous = decodedPayloads.get(envelopeId); + if (previous) decodedBytes -= previous.body.length; + decodedPayloads.set(envelopeId, payload); + decodedBytes += payload.body.length; + while ( + decodedPayloads.size > maxHistoryEntries || + decodedBytes > maxHistoryBytes + ) { + const oldest = decodedPayloads.keys().next().value!; + decodedBytes -= decodedPayloads.get(oldest)!.body.length; + decodedPayloads.delete(oldest); + } + } + + // Register before public listeners so decoded history commits with the core snapshot. + telemetry.subscribe((snapshot) => { + for (const envelope of snapshot.envelopes) { + const envelopeId = stringifyValue(envelope[0].__spotlight_envelope_id); + const payload = stagedPayloads.get(envelopeId); + if (!payload) continue; + stagedPayloads.delete(envelopeId); + retainDecodedPayload(envelopeId, payload); + } + }); + + function ignored(reason: IgnoreReason): IngestResult { + return receipt( + { status: "ignored", reason }, + { revision: telemetry.getStats().revision }, + ); + } + function invalid(code: DiagnosticCode, message: string): IngestResult { + const item = diagnostic(code, message); + reportDiagnostic(options.onDiagnostic, item); + return receipt( + { status: "invalid", diagnostics: [item] }, + { revision: telemetry.getStats().revision }, + ); + } + + async function processPayload( + input: TelemetryPayload, + body: Buffer, + contentType: string, + encoding: string | undefined, + currentGeneration: number, + ) { + function cancelled(): IngestResult | undefined { + if (disposed) return ignored("disposed"); + if (generation !== currentGeneration) return ignored("reset"); + if (input.signal?.aborted) return ignored("aborted"); + return undefined; + } + const initialCancellation = cancelled(); + if (initialCancellation) return initialCancellation; + if (encoding && encoding !== "identity") { + const controller = new AbortController(); + const abort = () => controller.abort(); + decodeControllers.add(controller); + input.signal?.addEventListener("abort", abort, { once: true }); + try { + // zlib applies this bound while producing output, before a compressed bomb can expand in memory. + const decodeOptions = { + maxOutputLength: maxDecodedBytes, + signal: controller.signal, + }; + body = await new Promise((resolve, reject) => { + const completed = (error: Error | null, output: Buffer) => { + if (error) reject(error); + else resolve(output); + }; + switch (encoding) { + case "gzip": + gunzip(body, decodeOptions, completed); + break; + case "deflate": + inflate(body, decodeOptions, completed); + break; + case "br": + brotliDecompress(body, decodeOptions, completed); + break; + default: + reject(new Error("Unsupported content encoding")); + } + }); + } catch (error) { + const cancellation = cancelled(); + if (cancellation) return cancellation; + if ( + error && + typeof error === "object" && + "code" in error && + error.code === "ERR_BUFFER_TOO_LARGE" + ) { + return invalid( + "decoded_payload_too_large", + "Decoded envelope exceeds maxDecodedBytes", + ); + } + return invalid( + "decompression_failed", + "Could not decompress the envelope body", + ); + } finally { + decodeControllers.delete(controller); + input.signal?.removeEventListener("abort", abort); + } + } + const cancellation = cancelled(); + if (cancellation) return cancellation; + if (body.length > maxDecodedBytes) + return invalid( + "decoded_payload_too_large", + "Decoded envelope exceeds maxDecodedBytes", + ); + let parseFailed = false; + let envelope: ReturnType = null; + try { + envelope = decodeEnvelope( + body, + input.userAgent, + () => { + parseFailed = true; + }, + true, + ); + } catch { + parseFailed = true; + } + if (!envelope || parseFailed) + return invalid( + "invalid_envelope", + "Malformed serialized Sentry envelope", + ); + // Normalize the codec's legacy UUID object before staging; the core publishes this exact string ID. + const envelopeId = + input.deliveryId ?? envelope[0].__spotlight_envelope_id.toString(); + const parsedEnvelope = envelope as Envelope; + parsedEnvelope[0].__spotlight_envelope_id = envelopeId; + const payload = { body, contentType }; + if ( + maxHistoryEntries > 0 && + maxHistoryBytes > 0 && + body.length <= maxHistoryBytes + ) + stagedPayloads.set(envelopeId, payload); + try { + const result = await withCoreAdmission(() => + telemetry.ingestEnvelope(parsedEnvelope, { + deliveryId: input.deliveryId, + signal: input.signal, + }), + ); + const retainPayload = + result.status === "processed" || + (result.status === "ignored" && result.reason === "unsupported_items"); + if ( + retainPayload && + "envelopeId" in result && + result.envelopeId && + !cancelled() && + stagedPayloads.get(envelopeId) === payload + ) { + // Raw history is independent of processed retention, including immediate age/byte eviction. + retainDecodedPayload(result.envelopeId, payload); + } + return result; + } finally { + if (stagedPayloads.get(envelopeId) === payload) + stagedPayloads.delete(envelopeId); + } + } + + async function withCoreAdmission(work: () => Promise): Promise { + const admission = Symbol(); + coreAdmissions.add(admission); + try { + return await work(); + } finally { + coreAdmissions.delete(admission); + } + } + + type PreparedEnvelope = { envelope: Envelope; options: IngestOptions }; + function prepareEnvelope( + input: unknown, + inputOptions: IngestOptions = {}, + ): PreparedEnvelope | IngestResult { + const captured = { + deliveryId: inputOptions.deliveryId, + signal: inputOptions.signal, + }; + if (captured.signal?.aborted) return ignored("aborted"); + if ( + captured.deliveryId !== undefined && + (typeof captured.deliveryId !== "string" || !captured.deliveryId.trim()) + ) { + return invalid( + "invalid_delivery_id", + "deliveryId must be a non-empty string.", + ); + } + if (!validateEnvelope(input)) { + return invalid( + "invalid_envelope", + "Expected a parsed envelope [headers, [[itemHeaders, payload], ...]].", + ); + } + try { + // Clones need an explicit stable ID: the core cannot associate a fresh clone with its original object. + const id = + captured.deliveryId ?? + stringifyValue( + input[0].__spotlight_envelope_id ?? + identities.get(input) ?? + generateUuidv4(), + ); + identities.set(input, id); + const envelope = normalizeEnvelope(input); + envelope[0].__spotlight_envelope_id = id; + if ( + Buffer.byteLength(JSON.stringify(envelope), "utf8") > maxEnvelopeBytes + ) { + return invalid( + "payload_too_large", + "Parsed envelope exceeds maxEnvelopeBytes.", + ); + } + return { envelope, options: captured }; + } catch { + return invalid( + "invalid_envelope", + "Envelope data must be cloneable wire data.", + ); + } + } + + function processEnvelope( + prepared: PreparedEnvelope, + epoch: number, + ): Promise { + if (disposed) return Promise.resolve(ignored("disposed")); + if (epoch !== generation) return Promise.resolve(ignored("reset")); + if (prepared.options.signal?.aborted) + return Promise.resolve(ignored("aborted")); + return withCoreAdmission(() => + telemetry.ingestEnvelope(prepared.envelope, prepared.options), + ); + } + + function pendingCount() { + // Core validation may synchronously notify before core admission increments its own count. + // Every admitted Node job still occupies its slot throughout that callback. + return Math.max( + pending.size, + telemetry.getStats().pending + pending.size - coreAdmissions.size, + ); + } + + type PreparedPayload = { + input: TelemetryPayload; + body: Buffer; + contentType: string; + encoding?: string; + }; + function preparePayload( + input: TelemetryPayload, + ): PreparedPayload | IngestResult { + if ( + !input || + (typeof input.body !== "string" && !nodeTypes.isUint8Array(input.body)) + ) { + return invalid( + "invalid_body", + "Envelope body must be a string or Uint8Array", + ); + } + if (input.signal?.aborted) return ignored("aborted"); + if ( + input.contentType !== undefined && + typeof input.contentType !== "string" + ) { + return invalid("invalid_content_type", "Content type must be a string"); + } + const contentType = (input.contentType ?? envelopeContentType) + .split(";")[0] + .trim() + .toLowerCase(); + if (contentType !== envelopeContentType) + return ignored("unsupported_content_type"); + if (input.encoding !== undefined && typeof input.encoding !== "string") { + return invalid( + "unsupported_encoding", + "Content encoding must be a string", + ); + } + const encoding = input.encoding?.trim().toLowerCase(); + if ( + encoding && + encoding !== "identity" && + !compressedEncodings.has(encoding) + ) { + return invalid( + "unsupported_encoding", + `Unsupported content encoding: ${encoding}`, + ); + } + const size = + typeof input.body === "string" + ? Buffer.byteLength(input.body, "utf8") + : input.body.byteLength; + if (size > maxPayloadBytes) + return invalid( + "payload_too_large", + "Envelope body exceeds maxPayloadBytes", + ); + if ((!encoding || encoding === "identity") && size > maxDecodedBytes) { + return invalid( + "decoded_payload_too_large", + "Decoded envelope exceeds maxDecodedBytes", + ); + } + const body = + typeof input.body === "string" + ? Buffer.from(input.body, "utf8") + : Buffer.from(input.body); + return { input: { ...input }, body, contentType, encoding }; + } + + function enqueue(work: (epoch: number) => Promise): Promise { + const epoch = generation; + const operation = queue.then(() => work(epoch)); + pending.add(operation); + queue = operation.then( + () => {}, + () => {}, + ); + const release = () => { + pending.delete(operation); + }; + void operation.then(release, release); + return operation; + } + + function ingestPayload(input: TelemetryPayload): Promise { + if (disposed) return Promise.resolve(ignored("disposed")); + if (input?.signal?.aborted) return Promise.resolve(ignored("aborted")); + if (pendingCount() >= maxPending) + return Promise.resolve(ignored("queue_full")); + // Standalone payload bytes and metadata are copied before the first await. + const prepared = preparePayload(input); + if ("status" in prepared) return Promise.resolve(prepared); + return enqueue((epoch) => + processPayload( + prepared.input, + prepared.body, + prepared.contentType, + prepared.encoding, + epoch, + ), + ); + } + + function ingestEnvelope( + input: unknown, + inputOptions: IngestOptions = {}, + ): Promise { + if (disposed) return Promise.resolve(ignored("disposed")); + if (inputOptions.signal?.aborted) + return Promise.resolve(ignored("aborted")); + if (pendingCount() >= maxPending) + return Promise.resolve(ignored("queue_full")); + // Snapshot standalone parsed data before enqueueing while retaining the original object's delivery identity. + const prepared = prepareEnvelope(input, inputOptions); + if ("status" in prepared) return Promise.resolve(prepared); + return enqueue((epoch) => processEnvelope(prepared, epoch)); + } + + function ingestEnvelopes( + inputs: Iterable, + batchOptions: BatchOptions = {}, + ): Promise { + const signal = batchOptions.signal; + const rejected = disposed + ? "disposed" + : pendingCount() >= maxPending + ? "queue_full" + : undefined; + if (rejected) + return Promise.resolve( + Object.freeze(Array.from(inputs, () => ignored(rejected))), + ); + return enqueue(async (epoch) => { + const results: IngestResult[] = []; + for (const input of inputs) { + if (disposed) results.push(ignored("disposed")); + else if (epoch !== generation) results.push(ignored("reset")); + else { + const prepared = prepareEnvelope(input, { signal }); + results.push( + "status" in prepared + ? prepared + : await processEnvelope(prepared, epoch), + ); + } + } + return Object.freeze(results); + }); + } + + return { + ...telemetry, + ingestEnvelope, + ingestEnvelopes, + ingestPayload, + ingestPayloads(inputs) { + const rejected = disposed + ? "disposed" + : pendingCount() >= maxPending + ? "queue_full" + : undefined; + const entries = Array.from(inputs); + if (rejected) + return Promise.resolve( + Object.freeze(entries.map(() => ignored(rejected))), + ); + // One admitted queue operation keeps every batch entry ahead of later payload calls. + // Prepare each entry when reached so a large batch cannot allocate unbounded byte copies. + return enqueue(async (epoch) => { + const results: IngestResult[] = []; + for (const input of entries) { + if (disposed) results.push(ignored("disposed")); + else if (epoch !== generation) results.push(ignored("reset")); + else { + const prepared = preparePayload(input); + results.push( + "status" in prepared + ? prepared + : await processPayload( + prepared.input, + prepared.body, + prepared.contentType, + prepared.encoding, + epoch, + ), + ); + } + } + return Object.freeze(results); + }); + }, + getDecodedPayload(envelopeId) { + const payload = decodedPayloads.get(envelopeId); + return ( + payload && { + body: Buffer.from(payload.body), + contentType: payload.contentType, + } + ); + }, + getStats() { + const stats = telemetry.getStats(); + return Object.freeze({ ...stats, pending: pendingCount() }); + }, + async drain() { + // Capture both barriers now: work submitted after this call cannot extend the drain. + const wireBarrier = queue; + const coreBarrier = telemetry.drain(); + await Promise.all([wireBarrier, coreBarrier]); + }, + setRetention(update) { + if (disposed) return Promise.resolve(); + const next = validateRetention(update); + const epoch = generation; + // Retention changes share payload ordering without consuming an admission slot. + const operation = queue.then(async () => { + if (!disposed && epoch === generation) + await telemetry.setRetention(next); + }); + queue = operation.catch(() => {}); + return operation; + }, + prune() { + if (disposed) return Promise.resolve(); + if (pruning) return pruning; + const epoch = generation; + // A single control continuation does not consume an ingestion admission slot. + const operation = queue.then(async () => { + if (!disposed && epoch === generation) await telemetry.prune(); + }); + queue = operation.catch(() => {}); + pruning = operation; + const release = () => { + if (pruning === operation) pruning = undefined; + }; + void operation.then(release, release); + return operation; + }, + reset() { + generation += 1; + pruning = undefined; + queue = Promise.resolve(); + pending.clear(); + coreAdmissions.clear(); + for (const controller of decodeControllers) controller.abort(); + decodedPayloads.clear(); + stagedPayloads.clear(); + decodedBytes = 0; + telemetry.reset(); + }, + dispose() { + disposed = true; + pruning = undefined; + generation += 1; + queue = Promise.resolve(); + pending.clear(); + coreAdmissions.clear(); + for (const controller of decodeControllers) controller.abort(); + decodedPayloads.clear(); + stagedPayloads.clear(); + decodedBytes = 0; + telemetry.dispose(); + }, + }; +} diff --git a/packages/telemetry/src/telemetry/ai/aiLibraries.ts b/packages/telemetry/src/telemetry/ai/aiLibraries.ts new file mode 100644 index 000000000..219f9306d --- /dev/null +++ b/packages/telemetry/src/telemetry/ai/aiLibraries.ts @@ -0,0 +1,62 @@ +import type { AILibraryHandler, AITrace, Span, Trace } from "../types.ts"; +import { genAIHandler } from "./genAI.ts"; +import { vercelAISDKHandler } from "./vercelAISDK.ts"; + +// Registry of AI span handlers. The generic gen_ai handler runs first because +// it only claims standard `gen_ai.*` spans that lack `vercel.ai.*` fields, so +// the Vercel handler still picks up its own proprietary spans. +const aiLibraries: AILibraryHandler[] = [genAIHandler, vercelAISDKHandler]; + +export { aiLibraries }; + +export function detectAILibraryHandler(span: Span): AILibraryHandler | null { + for (const handler of aiLibraries) { + // careful if we support multiple libraries, we should improve this method so it doesn't return the first one + if (handler.canHandleSpan(span)) { + return handler; + } + } + return null; +} + +export function extractAllAIRootSpans( + spans: Span[], +): { span: Span; handler: AILibraryHandler }[] { + const results: { span: Span; handler: AILibraryHandler }[] = []; + // Handlers can match overlapping span shapes (both AI handlers key off the + // `gen_ai.` op prefix). Attribute each root span to the first handler that + // claims it so a span is never surfaced twice. + const claimed = new Set(); + + for (const handler of aiLibraries) { + const rootSpans = handler.extractRootSpans(spans); + for (const rootSpan of rootSpans) { + if (claimed.has(rootSpan.span_id)) { + continue; + } + claimed.add(rootSpan.span_id); + results.push({ span: rootSpan, handler }); + } + } + + return results; +} + +export function createAITraceFromSpan(span: Span): AITrace | null { + const handler = detectAILibraryHandler(span); + if (!handler) { + return null; + } + + return handler.processTrace(span); +} + +export function hasAISpans(trace: Trace): boolean { + if (!trace.spanTree) { + return false; + } + + // TODO: We may want to cache/optimize this `extractAllAIRootSpans` helper + const aiRootSpans = extractAllAIRootSpans(trace.spanTree); + return aiRootSpans.length > 0; +} diff --git a/packages/telemetry/src/telemetry/ai/genAI.ts b/packages/telemetry/src/telemetry/ai/genAI.ts new file mode 100644 index 000000000..08e089748 --- /dev/null +++ b/packages/telemetry/src/telemetry/ai/genAI.ts @@ -0,0 +1,407 @@ +import { stringifyValue } from "../utils/stringifyValue.ts"; +import { getAllSpansInTree } from "../store/helpers.ts"; +import type { AILibraryHandler, AIToolCall, AITrace, Span } from "../types.ts"; + +// Generic handler for the standardized `gen_ai.*` span conventions. Any SDK +// that follows sentry-conventions (sentry-python's PydanticAI/OpenAI/Anthropic +// integrations, the OTel GenAI conventions, ...) maps through here regardless +// of provider — the provider is just the `gen_ai.system` attribute, not a +// reason to fork the handler. The Vercel AI SDK keeps its own handler only +// because it emits proprietary `vercel.ai.*` fields on top of `gen_ai.*`. +// https://github.com/getsentry/sentry-conventions/tree/main/model/attributes/gen_ai +const GEN_AI_OP_PREFIX = "gen_ai."; +const VERCEL_FIELD_PREFIX = "vercel.ai."; + +// Span ops +const GEN_AI_CHAT_OP = "gen_ai.chat"; +const GEN_AI_INVOKE_AGENT_OP = "gen_ai.invoke_agent"; +const GEN_AI_EXECUTE_TOOL_OP = "gen_ai.execute_tool"; + +// Span data fields (sentry-conventions gen_ai.*) +const GEN_AI_OPERATION_NAME_FIELD = "gen_ai.operation.name"; +const GEN_AI_AGENT_NAME_FIELD = "gen_ai.agent.name"; +const GEN_AI_SYSTEM_FIELD = "gen_ai.system"; +const GEN_AI_REQUEST_MODEL_FIELD = "gen_ai.request.model"; +const GEN_AI_RESPONSE_MODEL_FIELD = "gen_ai.response.model"; +const GEN_AI_USAGE_INPUT_TOKENS_FIELD = "gen_ai.usage.input_tokens"; +const GEN_AI_USAGE_OUTPUT_TOKENS_FIELD = "gen_ai.usage.output_tokens"; +const GEN_AI_REQUEST_MESSAGES_FIELD = "gen_ai.request.messages"; +const GEN_AI_SYSTEM_INSTRUCTIONS_FIELD = "gen_ai.system_instructions"; +const GEN_AI_RESPONSE_TEXT_FIELD = "gen_ai.response.text"; +const GEN_AI_RESPONSE_TOOL_CALLS_FIELD = "gen_ai.response.tool_calls"; +const GEN_AI_RESPONSE_FINISH_REASONS_FIELD = "gen_ai.response.finish_reasons"; +const GEN_AI_TOOL_NAME_FIELD = "gen_ai.tool.name"; +const GEN_AI_TOOL_CALL_ID_FIELD = "gen_ai.tool.call.id"; +const GEN_AI_TOOL_INPUT_FIELD = "gen_ai.tool.input"; +const GEN_AI_TOOL_OUTPUT_FIELD = "gen_ai.tool.output"; + +const DEFAULT_TRACE_NAME = "AI Interaction"; +const UNKNOWN_OPERATION = "N/A"; + +const OPERATION_BADGE_MAP = new Map([ + ["chat", "Chat"], + ["invoke_agent", "Invoke Agent"], + ["execute_tool", "Execute Tool"], + [GEN_AI_CHAT_OP, "Chat"], + [GEN_AI_INVOKE_AGENT_OP, "Invoke Agent"], + [GEN_AI_EXECUTE_TOOL_OP, "Execute Tool"], +]); + +function isGenAISpan(span: Span): boolean { + return !!span.op?.toLowerCase().startsWith(GEN_AI_OP_PREFIX); +} + +/** + * Claims any standard `gen_ai.*` span that isn't a Vercel AI SDK span. Both + * carry the `gen_ai.` op prefix, but only the Vercel SDK sets `vercel.ai.*` + * data fields, so we defer those to the Vercel handler. A generic gen_ai span + * has at least one `gen_ai.*` data key and no `vercel.ai.*` key. + */ +function isGenAIConventionSpan(span: Span): boolean { + if (!isGenAISpan(span) || !span.data) { + return false; + } + + let hasGenAIField = false; + for (const key of Object.keys(span.data)) { + if (key.startsWith(VERCEL_FIELD_PREFIX)) { + return false; + } + if (key.startsWith(GEN_AI_OP_PREFIX)) { + hasGenAIField = true; + } + } + + return hasGenAIField; +} + +export const genAIHandler: AILibraryHandler = { + id: "gen-ai", + name: "Gen AI", + + canHandleSpan: isGenAIConventionSpan, + + extractRootSpans: (spans: Span[]): Span[] => { + const resultRoots: Span[] = []; + + const findAndCollectAIRoots = (spansToSearch: Span[]) => { + for (const currentSpan of spansToSearch) { + if (isGenAIConventionSpan(currentSpan)) { + resultRoots.push(currentSpan); + } else if (!isGenAISpan(currentSpan) && currentSpan.children?.length) { + // Stop at any gen_ai.* span: it's the shallowest AI root for its + // tree (claimed by this handler or the Vercel one), so its nested + // gen_ai.* children must not be surfaced as separate roots. + findAndCollectAIRoots(currentSpan.children); + } + } + }; + + findAndCollectAIRoots(spans); + return resultRoots; + }, + + processTrace: (rootSpan: Span): AITrace => { + const allSpans = getAllSpansInTree(rootSpan); + const tokenUsage = extractTokenUsage(allSpans); + const operation = determineOperation(rootSpan); + const hasToolCall = allSpans.some( + (span) => + span.op === GEN_AI_EXECUTE_TOOL_OP || + !!span.data?.[GEN_AI_TOOL_NAME_FIELD], + ); + + const durationMs = rootSpan.timestamp - rootSpan.start_timestamp; + + const trace: AITrace = { + id: rootSpan.span_id, + name: determineTraceName(rootSpan), + operation, + timestamp: rootSpan.start_timestamp, + durationMs, + tokensDisplay: formatTokensDisplay( + tokenUsage.promptTokens, + tokenUsage.completionTokens, + ), + promptTokens: tokenUsage.promptTokens, + completionTokens: tokenUsage.completionTokens, + hasToolCall, + rawSpan: rootSpan, + metadata: { + metadata: {}, + promptTokens: tokenUsage.promptTokens, + completionTokens: tokenUsage.completionTokens, + }, + toolCalls: [], + }; + + const responseState = { finishReasonFromText: false }; + for (const span of allSpans) { + if (!span.data) continue; + extractMetadata(span, trace); + extractPromptData(span, trace); + extractResponseData(span, trace, responseState); + extractToolCallData(span, trace); + } + + return trace; + }, + + getDisplayTitle: (trace: AITrace): string => { + return ( + trace.name || trace.operation || `AI Trace ${trace.id.substring(0, 8)}` + ); + }, + + getTypeBadge: (trace: AITrace): string => { + if (trace.hasToolCall) { + return "Tool-Call"; + } + + return ( + OPERATION_BADGE_MAP.get(trace.operation) ?? + trace.operation.replace(/^gen_ai\./, "") + ); + }, + + getTokensDisplay: (trace: AITrace): string => { + return trace.tokensDisplay; + }, +}; + +function determineOperation(rootSpan: Span): string { + const operationName = rootSpan.data?.[GEN_AI_OPERATION_NAME_FIELD]; + if (operationName) { + return stringifyValue(operationName); + } + if (rootSpan.op) { + return rootSpan.op; + } + return UNKNOWN_OPERATION; +} + +function determineTraceName(rootSpan: Span): string { + const agentName = rootSpan.data?.[GEN_AI_AGENT_NAME_FIELD]; + if (agentName) { + return stringifyValue(agentName); + } + return rootSpan.description || rootSpan.op || DEFAULT_TRACE_NAME; +} + +/** + * Sums token usage across every span in the tree so an agent root with multiple + * chat turns reports the total, not just the first turn's usage. + */ +function extractTokenUsage(spans: Span[]): { + promptTokens?: number; + completionTokens?: number; +} { + let promptTokens: number | undefined; + let completionTokens: number | undefined; + + for (const span of spans) { + if (!span.data) continue; + + if (span.data[GEN_AI_USAGE_INPUT_TOKENS_FIELD] !== undefined) { + promptTokens = + (promptTokens ?? 0) + + Number(span.data[GEN_AI_USAGE_INPUT_TOKENS_FIELD]); + } + + if (span.data[GEN_AI_USAGE_OUTPUT_TOKENS_FIELD] !== undefined) { + completionTokens = + (completionTokens ?? 0) + + Number(span.data[GEN_AI_USAGE_OUTPUT_TOKENS_FIELD]); + } + } + + return { promptTokens, completionTokens }; +} + +function formatTokensDisplay( + promptTokens?: number, + completionTokens?: number, +): string { + if (promptTokens !== undefined && completionTokens !== undefined) { + return `${promptTokens} / ${completionTokens}`; + } + if (promptTokens !== undefined) { + return `${promptTokens} / ?`; + } + if (completionTokens !== undefined) { + return `? / ${completionTokens}`; + } + return UNKNOWN_OPERATION; +} + +/** + * Normalizes message content from a string or an array of content blocks to a + * plain string. Only text blocks are surfaced; images/files would need UI work. + */ +function normalizeMessageContent(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .filter( + (block): block is { type: string; text: string } => + block?.type === "text" && typeof block.text === "string", + ) + .map((block) => block.text) + .join(""); + } + return ""; +} + +function extractMetadata(span: Span, trace: AITrace) { + if (!span.data) return; + + if (trace.metadata.modelId === undefined) { + const modelId = + span.data[GEN_AI_REQUEST_MODEL_FIELD] ?? + span.data[GEN_AI_RESPONSE_MODEL_FIELD]; + if (modelId !== undefined) { + trace.metadata.modelId = stringifyValue(modelId); + } + } + + if ( + trace.metadata.modelProvider === undefined && + span.data[GEN_AI_SYSTEM_FIELD] !== undefined + ) { + trace.metadata.modelProvider = stringifyValue( + span.data[GEN_AI_SYSTEM_FIELD], + ); + } +} + +function extractPromptData(span: Span, trace: AITrace) { + if (!span.data || trace.prompt) return; + + const promptMessages = span.data[GEN_AI_REQUEST_MESSAGES_FIELD]; + if (promptMessages === undefined) return; + + const system = span.data[GEN_AI_SYSTEM_INSTRUCTIONS_FIELD]; + const systemText = system !== undefined ? stringifyValue(system) : undefined; + + try { + const messages = + typeof promptMessages === "string" + ? JSON.parse(promptMessages) + : promptMessages; + if (Array.isArray(messages)) { + const normalizedMessages = messages.map( + (msg: { role: string; content: unknown }) => ({ + ...msg, + content: normalizeMessageContent(msg.content), + }), + ); + trace.prompt = { messages: normalizedMessages, system: systemText }; + return; + } + } catch { + // fall through to the raw representation below + } + + trace.prompt = { + messages: [{ role: "unknown", content: stringifyValue(promptMessages) }], + system: systemText, + }; +} + +function extractFinishReason(finishReasons: unknown): string | undefined { + if (Array.isArray(finishReasons) && finishReasons.length > 0) { + return stringifyValue(finishReasons[0]); + } + if (typeof finishReasons === "string" && finishReasons) { + return finishReasons; + } + return undefined; +} + +function extractResponseData( + span: Span, + trace: AITrace, + state: { finishReasonFromText: boolean }, +) { + if (!span.data) return; + + trace.response = trace.response || {}; + + // Spans arrive in tree pre-order, so for an agent root with several chat + // children the last span carrying a response wins — surfacing the latest + // model turn rather than the first. + const responseText = span.data[GEN_AI_RESPONSE_TEXT_FIELD]; + const text = Array.isArray(responseText) + ? responseText.map(stringifyValue).join("") + : responseText + ? stringifyValue(responseText) + : ""; + + const finishReason = extractFinishReason( + span.data[GEN_AI_RESPONSE_FINISH_REASONS_FIELD], + ); + + if (text) { + trace.response.text = text; + // Re-pair the finish reason with the turn that produced the surfaced text, + // even overwriting one an earlier text-less turn contributed, so the text + // and its reason always come from the same turn. + trace.response.finishReason = finishReason; + state.finishReasonFromText = true; + } else if (finishReason !== undefined && !state.finishReasonFromText) { + // SDKs omit gen_ai.response.text when PII recording is off, and tool-call + // turns carry a reason with no text — still surface the reason, but never + // let it override one already paired with a text turn. + trace.response.finishReason = finishReason; + } + + const toolCalls = span.data[GEN_AI_RESPONSE_TOOL_CALLS_FIELD]; + if (toolCalls) { + try { + trace.response.toolCalls = + typeof toolCalls === "string" + ? JSON.parse(toolCalls) + : (toolCalls as AIToolCall[]); + } catch { + // ignore unparseable tool call payloads + } + } +} + +function extractToolCallData(span: Span, trace: AITrace) { + if (!span.data) return; + + const toolName = span.data[GEN_AI_TOOL_NAME_FIELD]; + if (!toolName) return; + + const toolCallId = span.data[GEN_AI_TOOL_CALL_ID_FIELD]; + const toolCall: Partial = { + toolCallId: toolCallId ? stringifyValue(toolCallId) : span.span_id, + toolName: stringifyValue(toolName), + args: {} as Record, + }; + + const toolInput = span.data[GEN_AI_TOOL_INPUT_FIELD]; + if (toolInput) { + try { + toolCall.args = + typeof toolInput === "string" + ? JSON.parse(toolInput) + : (toolInput as Record); + } catch { + toolCall.args = { input: toolInput }; + } + } + + const toolOutput = span.data[GEN_AI_TOOL_OUTPUT_FIELD]; + if (toolOutput) { + try { + toolCall.result = + typeof toolOutput === "string" ? JSON.parse(toolOutput) : toolOutput; + } catch { + toolCall.result = stringifyValue(toolOutput); + } + } + + trace.toolCalls.push(toolCall as AIToolCall); +} diff --git a/packages/telemetry/src/telemetry/ai/inspection.ts b/packages/telemetry/src/telemetry/ai/inspection.ts new file mode 100644 index 000000000..91d96fd52 --- /dev/null +++ b/packages/telemetry/src/telemetry/ai/inspection.ts @@ -0,0 +1,137 @@ +import { record, parsed } from "../internal/wireValues.ts"; +/** Merge legacy span data and typed span-v2 attributes without losing unknown fields. */ +export function readSpanData(payload: unknown): Record { + const p = record(payload); + const attributes = Object.fromEntries( + Object.entries(record(p.attributes)).map(([key, value]) => [ + key, + "value" in record(value) ? record(value).value : value, + ]), + ); + return { + ...record(record(record(p.contexts).trace).data), + ...record(p.data), + ...attributes, + }; +} +export function isAIInteraction(payload: unknown): boolean { + const p = record(payload), + data = readSpanData(p); + return ( + [ + p.type, + p.op, + record(record(p.contexts).trace).op, + data["sentry.op"], + data["sentry.origin"], + ].some( + (value) => + typeof value === "string" && + /(?:^|[.\s:_-])(?:ai|gen_ai|llm)(?:$|[.\s:_-])/i.test(value), + ) || + Object.keys(data).some((key) => /^(?:gen_ai|vercel\.ai|ai|llm)\./.test(key)) + ); +} +export type AIInspection = { + fields: Record; + system?: unknown; + messages?: unknown; + response?: unknown; + tools: unknown[]; +}; +export function inspectAI(payload: unknown): AIInspection | undefined { + if (!isAIInteraction(payload)) return; + const data = readSpanData(payload); + const pick = (...keys: string[]) => + keys + .map((key) => data[key]) + .find((value) => value !== undefined && value !== null); + const definitions: [string, string[]][] = [ + [ + "model", + ["gen_ai.request.model", "gen_ai.response.model", "vercel.ai.model.id"], + ], + [ + "provider", + ["gen_ai.system", "gen_ai.provider.name", "vercel.ai.model.provider"], + ], + [ + "inputTokens", + [ + "gen_ai.usage.input_tokens", + "vercel.ai.usage.promptTokens", + "ai.usage.promptTokens", + ], + ], + [ + "outputTokens", + [ + "gen_ai.usage.output_tokens", + "vercel.ai.usage.completionTokens", + "ai.usage.completionTokens", + ], + ], + [ + "finishReason", + ["gen_ai.response.finish_reasons", "vercel.ai.response.finishReason"], + ], + ["function", ["gen_ai.function_id", "vercel.ai.telemetry.functionId"]], + ["maxRetries", ["vercel.ai.settings.maxRetries"]], + ["maxSteps", ["vercel.ai.settings.maxSteps"]], + ]; + const calls = parsed( + pick( + "gen_ai.response.tool_calls", + "vercel.ai.response.toolCalls", + "ai.response.toolCalls", + ), + ); + const tools = Array.isArray(calls) + ? [...calls] + : calls === undefined + ? [] + : [calls]; + const toolName = pick("gen_ai.tool.name", "ai.toolCall.name"); + if (toolName !== undefined) + tools.push({ + name: toolName, + id: pick("gen_ai.tool.call.id", "ai.toolCall.id"), + arguments: parsed(pick("gen_ai.tool.input", "ai.toolCall.args")), + result: parsed(pick("gen_ai.tool.output", "ai.toolCall.result")), + }); + return { + fields: Object.fromEntries( + definitions.flatMap(([label, keys]) => { + const value = pick(...keys); + return value === undefined ? [] : [[label, value]]; + }), + ), + system: parsed( + pick( + "gen_ai.system_instructions", + "gen_ai.prompt.system", + "ai.prompt.system", + ), + ), + messages: parsed( + pick( + "gen_ai.request.messages", + "gen_ai.input.messages", + "vercel.ai.prompt.messages", + "vercel.ai.prompt", + "gen_ai.prompt", + "ai.prompt.messages", + "ai.prompt", + ), + ), + response: parsed( + pick( + "gen_ai.response.text", + "gen_ai.output.messages", + "vercel.ai.response.text", + "ai.response.text", + ), + ), + tools, + }; +} diff --git a/packages/telemetry/src/telemetry/ai/vercelAISDK.ts b/packages/telemetry/src/telemetry/ai/vercelAISDK.ts new file mode 100644 index 000000000..3d20a8169 --- /dev/null +++ b/packages/telemetry/src/telemetry/ai/vercelAISDK.ts @@ -0,0 +1,536 @@ +import { stringifyValue } from "../utils/stringifyValue.ts"; +import { getAllSpansInTree } from "../store/helpers.ts"; +import type { AILibraryHandler, AIToolCall, AITrace, Span } from "../types.ts"; + +// https://ai-sdk.dev/docs/ai-sdk-core/telemetry +const AI_SPAN_OP_PREFIX = "gen_ai."; +const AI_OPERATION_ID_FIELD = "vercel.ai.operationId"; +const AI_OPERATION_NAME_FIELD = "operation.name"; + +// Tool call related constants (OpenTelemetry semantic conventions) +const AI_TOOL_CALL_OPERATION = "ai.toolCall"; +const AI_EXECUTE_TOOL_OPERATION = "gen_ai.execute_tool"; +const GEN_AI_TOOL_NAME_FIELD = "gen_ai.tool.name"; +const GEN_AI_TOOL_CALL_ID_FIELD = "gen_ai.tool.call.id"; +const GEN_AI_TOOL_INPUT_FIELD = "gen_ai.tool.input"; +const GEN_AI_TOOL_OUTPUT_FIELD = "gen_ai.tool.output"; + +const AI_MODEL_ID_FIELD = "vercel.ai.model.id"; +const AI_MODEL_PROVIDER_FIELD = "vercel.ai.model.provider"; +const AI_SETTINGS_MAX_RETRIES_FIELD = "vercel.ai.settings.maxRetries"; +const AI_SETTINGS_MAX_STEPS_FIELD = "vercel.ai.settings.maxSteps"; +const AI_TELEMETRY_FUNCTION_ID_FIELD = "vercel.ai.telemetry.functionId"; +const AI_TELEMETRY_METADATA_PREFIX = "vercel.ai.telemetry.metadata."; +const AI_PROMPT_FIELD = "vercel.ai.prompt"; +const AI_PROMPT_MESSAGES_FIELD = "gen_ai.request.messages"; +const AI_RESPONSE_FINISH_REASON_FIELD = "vercel.ai.response.finishReason"; +const GEN_AI_RESPONSE_FINISH_REASONS_FIELD = "gen_ai.response.finish_reasons"; +const AI_RESPONSE_TEXT_FIELD = "gen_ai.response.text"; +const AI_RESPONSE_TOOL_CALLS_FIELD = "vercel.ai.response.toolCalls"; + +// v1 operation names +const AI_STREAM_TEXT_OPERATION = "ai.streamText"; +const AI_GENERATE_TEXT_OPERATION = "ai.generateText"; + +// v2 operation names (OpenTelemetry semantic conventions) +const AI_INVOKE_AGENT_OPERATION = "gen_ai.invoke_agent"; +const AI_STREAM_TEXT_V2_OPERATION = "gen_ai.stream_text"; + +// v2 field names +const GEN_AI_PROMPT_FIELD = "gen_ai.prompt"; +const GEN_AI_FUNCTION_ID_FIELD = "gen_ai.function_id"; +const GEN_AI_REQUEST_MODEL_FIELD = "gen_ai.request.model"; +const GEN_AI_RESPONSE_MODEL_FIELD = "gen_ai.response.model"; +const GEN_AI_SYSTEM_FIELD = "gen_ai.system"; +const GEN_AI_OPERATION_NAME_FIELD = "gen_ai.operation.name"; + +const AI_USAGE_PROMPT_TOKENS_FIELD = "vercel.ai.usage.promptTokens"; +const AI_USAGE_COMPLETION_TOKENS_FIELD = "vercel.ai.usage.completionTokens"; +const GEN_AI_USAGE_INPUT_TOKENS_FIELD = "gen_ai.usage.input_tokens"; +const GEN_AI_USAGE_OUTPUT_TOKENS_FIELD = "gen_ai.usage.output_tokens"; + +const TOKEN_FIELDS = { + PROMPT: [AI_USAGE_PROMPT_TOKENS_FIELD, GEN_AI_USAGE_INPUT_TOKENS_FIELD], + COMPLETION: [ + AI_USAGE_COMPLETION_TOKENS_FIELD, + GEN_AI_USAGE_OUTPUT_TOKENS_FIELD, + ], +} as const; + +// Other constants +const DEFAULT_TRACE_NAME = "AI Interaction"; +const UNKNOWN_OPERATION = "N/A"; + +// Operation to badge mapping +const OPERATION_BADGE_MAP = new Map([ + [AI_STREAM_TEXT_OPERATION, "Stream Text"], + [AI_GENERATE_TEXT_OPERATION, "Generate Text"], + [AI_INVOKE_AGENT_OPERATION, "Invoke Agent"], + [AI_STREAM_TEXT_V2_OPERATION, "Stream Text"], +]); + +// Field mappings: first element is v2, subsequent are v1 alternatives +const AI_FIELD_MAPPINGS = { + MODEL_ID: [ + AI_MODEL_ID_FIELD, + GEN_AI_REQUEST_MODEL_FIELD, + GEN_AI_RESPONSE_MODEL_FIELD, + ], + MODEL_PROVIDER: [AI_MODEL_PROVIDER_FIELD, GEN_AI_SYSTEM_FIELD], + FUNCTION_ID: [AI_TELEMETRY_FUNCTION_ID_FIELD, GEN_AI_FUNCTION_ID_FIELD], + PROMPT_TOKENS: [ + AI_USAGE_PROMPT_TOKENS_FIELD, + GEN_AI_USAGE_INPUT_TOKENS_FIELD, + ], + COMPLETION_TOKENS: [ + AI_USAGE_COMPLETION_TOKENS_FIELD, + GEN_AI_USAGE_OUTPUT_TOKENS_FIELD, + ], + PROMPT: [GEN_AI_PROMPT_FIELD, AI_PROMPT_FIELD], + OPERATION_ID: [ + AI_OPERATION_ID_FIELD, + AI_OPERATION_NAME_FIELD, + GEN_AI_OPERATION_NAME_FIELD, + ], +} as const; + +/** + * Extracts a field value from span data, checking v1 fields first then v2 alternatives. + */ +function extractFieldFromSpan( + data: Record, + fieldKey: keyof typeof AI_FIELD_MAPPINGS, +): unknown { + const fields = AI_FIELD_MAPPINGS[fieldKey]; + for (const field of fields) { + if (data[field] !== undefined) { + return data[field]; + } + } + return undefined; +} + +export const vercelAISDKHandler: AILibraryHandler = { + id: "vercel-ai-sdk", + name: "Vercel AI SDK", + + canHandleSpan: (span: Span): boolean => { + return !!span.op?.toLowerCase().startsWith(AI_SPAN_OP_PREFIX); + }, + + extractRootSpans: (spans: Span[]): Span[] => { + const resultRoots: Span[] = []; + + const findAndCollectAIRoots = (spansToSearch: Span[]) => { + for (const currentSpan of spansToSearch) { + if (currentSpan.op?.toLowerCase().startsWith(AI_SPAN_OP_PREFIX)) { + resultRoots.push(currentSpan); + } else if (currentSpan.children?.length) { + findAndCollectAIRoots(currentSpan.children); + } + } + }; + + findAndCollectAIRoots(spans); + return resultRoots; + }, + + processTrace: (rootSpan: Span): AITrace => { + const allSpans = getAllSpansInTree(rootSpan); + const tokenUsage = extractTokenUsage(allSpans); + const operationInfo = determineOperation(allSpans, rootSpan); + const traceName = determineTraceName(rootSpan, operationInfo); + + const durationMs = rootSpan.timestamp - rootSpan.start_timestamp; + + // Initialize the trace with basic properties + const trace: AITrace = { + id: rootSpan.span_id, + name: traceName, + operation: operationInfo.operation, + timestamp: rootSpan.start_timestamp, + durationMs, + tokensDisplay: formatTokensDisplay( + tokenUsage.promptTokens, + tokenUsage.completionTokens, + ), + promptTokens: tokenUsage.promptTokens, + completionTokens: tokenUsage.completionTokens, + hasToolCall: operationInfo.hasToolCall, + rawSpan: rootSpan, + metadata: { + metadata: {}, + }, + toolCalls: [], + }; + + // Parse detailed span data + parseSpanData(allSpans, trace); + + return trace; + }, + + getDisplayTitle: (trace: AITrace): string => { + return ( + trace.name || trace.operation || `AI Trace ${trace.id.substring(0, 8)}` + ); + }, + + getTypeBadge: (trace: AITrace): string => { + if (trace.hasToolCall) { + return "Tool-Call"; + } + + return ( + OPERATION_BADGE_MAP.get(trace.operation) ?? + trace.operation.replace(/^(ai\.|gen_ai\.)/, "") + ); + }, + + getTokensDisplay: (trace: AITrace): string => { + return trace.tokensDisplay; + }, +}; + +function extractTokenUsage(spans: Span[]): { + promptTokens?: number; + completionTokens?: number; +} { + let promptTokens: number | undefined; + let completionTokens: number | undefined; + + for (const span of spans) { + if (!span.data) continue; + + if (promptTokens === undefined) { + for (const field of TOKEN_FIELDS.PROMPT) { + const value = span.data[field]; + if (value !== undefined) { + promptTokens = Number(value); + break; + } + } + } + + if (completionTokens === undefined) { + for (const field of TOKEN_FIELDS.COMPLETION) { + const value = span.data[field]; + if (value !== undefined) { + completionTokens = Number(value); + break; + } + } + } + + if (promptTokens !== undefined && completionTokens !== undefined) { + break; + } + } + + return { promptTokens, completionTokens }; +} + +function determineOperation( + spans: Span[], + rootSpan: Span, +): { operation: string; hasToolCall: boolean; toolCallName?: string } { + let operation = UNKNOWN_OPERATION; + let hasToolCall = false; + let toolCallName: string | undefined; + let foundToolCallAsOperationId = false; + + for (const span of spans) { + if ( + span.op === AI_TOOL_CALL_OPERATION || + span.op === AI_EXECUTE_TOOL_OPERATION + ) { + hasToolCall = true; + } + + if (!span.data) continue; + + const operationId = extractFieldFromSpan(span.data, "OPERATION_ID") as + | string + | undefined; + + // Handle tool call operation + if (operationId === AI_TOOL_CALL_OPERATION) { + operation = AI_TOOL_CALL_OPERATION; + foundToolCallAsOperationId = true; + hasToolCall = true; + + if (span.data[GEN_AI_TOOL_NAME_FIELD]) { + toolCallName = stringifyValue(span.data[GEN_AI_TOOL_NAME_FIELD]); + } + } + + // Handle other operations (prioritize root and direct children) + if ( + !foundToolCallAsOperationId && + operationId && + operation === UNKNOWN_OPERATION + ) { + const isHighPriority = + span.span_id === rootSpan.span_id || + rootSpan.children?.some((child) => child.span_id === span.span_id); + + if (isHighPriority) { + operation = operationId; + } + } + } + + // Fallback logic + if (operation === UNKNOWN_OPERATION) { + if (rootSpan.op) { + operation = rootSpan.op; + } else if (rootSpan.description) { + operation = rootSpan.description; + } + } + + return { operation, hasToolCall, toolCallName }; +} + +function determineTraceName( + rootSpan: Span, + operationInfo: { toolCallName?: string }, +): string { + if (operationInfo.toolCallName) { + return operationInfo.toolCallName; + } + + return rootSpan.description || rootSpan.op || DEFAULT_TRACE_NAME; +} + +function formatTokensDisplay( + promptTokens?: number, + completionTokens?: number, +): string { + if (promptTokens !== undefined && completionTokens !== undefined) { + return `${promptTokens} / ${completionTokens}`; + } + if (promptTokens !== undefined) { + return `${promptTokens} / ?`; + } + if (completionTokens !== undefined) { + return `? / ${completionTokens}`; + } + return UNKNOWN_OPERATION; +} + +/** + * Normalizes message content from v1 (string) or v2 (array of content blocks) format to a string. + * v1: content = "text string" + * v2: content = [{ type: "text", text: "text string" }] + * + * Only extracts text content blocks. Other block types (image, file, tool-call) + * would require UI changes to display properly and are filtered out for now. + */ +function normalizeMessageContent(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + // Only extract text content - images/files would need UI changes to display + return content + .filter( + (block): block is { type: string; text: string } => + block.type === "text" && typeof block.text === "string", + ) + .map((block) => block.text) + .join(""); + } + return ""; +} + +function parseSpanData(spans: Span[], trace: AITrace) { + for (const span of spans) { + if (!span.data) continue; + + extractAIMetadata(span, trace); + extractTelemetryMetadata(span, trace); + extractPromptData(span, trace); + extractResponseData(span, trace); + extractToolCallData(span, trace); + } +} + +function extractAIMetadata(span: Span, trace: AITrace) { + if (!span.data) return; + + // Model ID + if (trace.metadata.modelId === undefined) { + const modelId = extractFieldFromSpan(span.data, "MODEL_ID"); + if (modelId !== undefined) { + trace.metadata.modelId = stringifyValue(modelId); + } + } + + // Model Provider + if (trace.metadata.modelProvider === undefined) { + const provider = extractFieldFromSpan(span.data, "MODEL_PROVIDER"); + if (provider !== undefined) { + trace.metadata.modelProvider = stringifyValue(provider); + } + } + + // Function ID + if (trace.metadata.functionId === undefined) { + const functionId = extractFieldFromSpan(span.data, "FUNCTION_ID"); + if (functionId !== undefined) { + trace.metadata.functionId = stringifyValue(functionId); + } + } + + if (span.data[AI_SETTINGS_MAX_RETRIES_FIELD]) { + trace.metadata.maxRetries = Number( + span.data[AI_SETTINGS_MAX_RETRIES_FIELD], + ); + } + + if (span.data[AI_SETTINGS_MAX_STEPS_FIELD]) { + trace.metadata.maxSteps = Number(span.data[AI_SETTINGS_MAX_STEPS_FIELD]); + } + + // Token usage - use === undefined to allow 0 values + if (trace.metadata.promptTokens === undefined) { + const promptTokens = extractFieldFromSpan(span.data, "PROMPT_TOKENS"); + if (promptTokens !== undefined) { + trace.metadata.promptTokens = Number(promptTokens); + } + } + + if (trace.metadata.completionTokens === undefined) { + const completionTokens = extractFieldFromSpan( + span.data, + "COMPLETION_TOKENS", + ); + if (completionTokens !== undefined) { + trace.metadata.completionTokens = Number(completionTokens); + } + } +} + +function extractTelemetryMetadata(span: Span, trace: AITrace) { + if (!span.data) return; + + for (const [key, value] of Object.entries(span.data)) { + if (key.startsWith(AI_TELEMETRY_METADATA_PREFIX)) { + const metadataKey = key.replace(AI_TELEMETRY_METADATA_PREFIX, ""); + trace.metadata.metadata[metadataKey] = value; + } + } +} + +function extractPromptData(span: Span, trace: AITrace) { + if (!span.data) return; + + const promptField = extractFieldFromSpan(span.data, "PROMPT"); + if (promptField && !trace.prompt) { + try { + const parsed = JSON.parse(stringifyValue(promptField)); + // Normalize message content from v2 array format to string + if (parsed.messages && Array.isArray(parsed.messages)) { + parsed.messages = parsed.messages.map( + (msg: { role: string; content: unknown }) => ({ + ...msg, + content: normalizeMessageContent(msg.content), + }), + ); + } + trace.prompt = parsed; + } catch { + trace.prompt = { + messages: [{ role: "unknown", content: stringifyValue(promptField) }], + }; + } + } + + const promptMessages = span.data[AI_PROMPT_MESSAGES_FIELD]; + if (promptMessages && !trace.prompt) { + try { + const messages = JSON.parse(stringifyValue(promptMessages)); + // Normalize message content from v2 array format to string + const normalizedMessages = messages.map( + (msg: { role: string; content: unknown }) => ({ + ...msg, + content: normalizeMessageContent(msg.content), + }), + ); + trace.prompt = { messages: normalizedMessages }; + } catch { + trace.prompt = { + messages: [ + { role: "unknown", content: stringifyValue(promptMessages) }, + ], + }; + } + } +} + +function extractResponseData(span: Span, trace: AITrace) { + if (!span.data) return; + + trace.response = trace.response || {}; + + const finishReason = span.data[AI_RESPONSE_FINISH_REASON_FIELD]; + if (finishReason) { + trace.response.finishReason = stringifyValue(finishReason); + } + + const finishReasons = span.data[GEN_AI_RESPONSE_FINISH_REASONS_FIELD]; + if ( + !trace.response.finishReason && + Array.isArray(finishReasons) && + finishReasons.length > 0 + ) { + trace.response.finishReason = stringifyValue(finishReasons[0]); + } + + const responseText = span.data[AI_RESPONSE_TEXT_FIELD]; + if (responseText) { + trace.response.text = stringifyValue(responseText); + } + + const toolCalls = span.data[AI_RESPONSE_TOOL_CALLS_FIELD]; + if (toolCalls) { + trace.response.toolCalls = JSON.parse(stringifyValue(toolCalls)); + } +} + +function extractToolCallData(span: Span, trace: AITrace) { + if (!span.data) return; + + const toolName = span.data[GEN_AI_TOOL_NAME_FIELD]; + if (!toolName) return; + + const toolCallId = span.data[GEN_AI_TOOL_CALL_ID_FIELD]; + const toolCall: Partial = { + toolCallId: toolCallId ? stringifyValue(toolCallId) : span.span_id, + toolName: stringifyValue(toolName), + args: {} as Record, + }; + + const toolInput = span.data[GEN_AI_TOOL_INPUT_FIELD]; + if (toolInput) { + try { + toolCall.args = + typeof toolInput === "string" + ? JSON.parse(toolInput) + : (toolInput as Record); + } catch { + toolCall.args = { input: toolInput }; + } + } + + const toolOutput = span.data[GEN_AI_TOOL_OUTPUT_FIELD]; + if (toolOutput) { + try { + toolCall.result = + typeof toolOutput === "string" ? JSON.parse(toolOutput) : toolOutput; + } catch { + toolCall.result = stringifyValue(toolOutput); + } + } + + trace.toolCalls.push(toolCall as AIToolCall); +} diff --git a/packages/telemetry/src/telemetry/attachments.spec.ts b/packages/telemetry/src/telemetry/attachments.spec.ts new file mode 100644 index 000000000..42554d861 --- /dev/null +++ b/packages/telemetry/src/telemetry/attachments.spec.ts @@ -0,0 +1,87 @@ +import { describe, expect, test } from "vitest"; +import { createTelemetryStore } from "../sdk/core.ts"; + +describe("parsed protocol attachments", () => { + test.each([ + [ + "binary bytes", + new Uint8Array([0, 10, 255]), + "application/octet-stream", + "AAr/", + ], + ["binary string", "héllo", "application/octet-stream", "aMOpbGxv"], + [ + "text bytes", + new TextEncoder().encode("héllo\n世界"), + "text/plain", + "héllo\n世界", + ], + ["text string", "héllo\n世界", "text/plain", "héllo\n世界"], + [ + "text with byte order mark", + new Uint8Array([239, 187, 191, 65]), + "text/plain", + "\uFEFFA", + ], + ] as const)( + "associates %s with its event and retains canonical immutable data", + async (_, payload, contentType, data) => { + const telemetry = createTelemetryStore(); + const input = [ + {}, + [ + [ + { type: "event" }, + { event_id: "attached-event", message: "hello", timestamp: 1 }, + ], + [ + { type: "attachment", filename: "file", content_type: contentType }, + payload, + ], + ], + ]; + const original = structuredClone(input); + const result = await telemetry.ingestEnvelope(input); + expect(result.status).toBe("processed"); + expect(telemetry.getEvent("attached-event")?.attachments).toMatchObject([ + { data }, + ]); + expect(telemetry.getEnvelopes()[0][1][1][1]).toEqual({ data }); + expect(Object.isFrozen(telemetry.getEnvelopes()[0][1][1][1])).toBe(true); + expect(input).toEqual(original); + telemetry.dispose(); + }, + ); + + test("retains opaque unsupported items without discarding their accompanying event", async () => { + const telemetry = createTelemetryStore(); + const result = await telemetry.ingestEnvelope([ + {}, + [ + [{ type: "event" }, { event_id: "event", timestamp: 1 }], + [{ type: "future_binary" }, new Uint8Array([0, 255])], + [{ type: "future_text" }, "hello"], + ], + ]); + expect(result.status).toBe("processed"); + expect(telemetry.getEvent("event")).toBeDefined(); + expect( + telemetry + .getEnvelopes()[0][1] + .slice(1) + .map((item) => item[1]), + ).toEqual([{ data: "AP8=" }, { data: "aGVsbG8=" }]); + telemetry.dispose(); + }); + + test("continues to reject primitive payloads for supported telemetry items", async () => { + const telemetry = createTelemetryStore(); + const result = await telemetry.ingestEnvelope([ + {}, + [[{ type: "event" }, "not a parsed event"]], + ]); + expect(result.status).toBe("invalid"); + expect(telemetry.getEnvelopes()).toEqual([]); + telemetry.dispose(); + }); +}); diff --git a/packages/telemetry/src/telemetry/capabilities.ts b/packages/telemetry/src/telemetry/capabilities.ts new file mode 100644 index 000000000..9df693d2b --- /dev/null +++ b/packages/telemetry/src/telemetry/capabilities.ts @@ -0,0 +1,18 @@ +/** Wire formats understood by the portable SDK. This describes processing, not transport codecs. */ +export const SDK_CAPABILITIES = Object.freeze({ + schemaVersion: 1 as const, + spanFormats: Object.freeze(["legacy", "v2"] as const), + profileFormats: Object.freeze(["v1", "v2"] as const), + itemTypes: Object.freeze([ + "event", + "error", + "transaction", + "span", + "log", + "profile", + "profile_chunk", + "feedback", + "attachment", + ] as const), + timestampUnit: "unix-milliseconds" as const, +}); diff --git a/packages/telemetry/src/telemetry/client.spec.ts b/packages/telemetry/src/telemetry/client.spec.ts new file mode 100644 index 000000000..140b7cf27 --- /dev/null +++ b/packages/telemetry/src/telemetry/client.spec.ts @@ -0,0 +1,488 @@ +// @vitest-environment node +import { afterEach, expect, test, vi } from "vitest"; +import * as core from "./index.ts"; + +function spans(id: string, traceId = "trace", count = 1) { + return [ + { + __spotlight_envelope_id: id, + sdk: { name: "sentry.javascript.node", version: "11" }, + }, + [ + [ + { type: "span" }, + { + version: 2, + items: Array.from({ length: count }, (_, index) => ({ + trace_id: traceId, + span_id: `${id}-${index}`, + name: "SELECT 1", + start_timestamp: 1, + end_timestamp: 1.125, + attributes: { "sentry.op": { value: "db.query", type: "string" } }, + })), + }, + ], + ], + ]; +} +afterEach(() => vi.useRealTimers()); + +test("canonical reads have consistent filters and explicit summary units", async () => { + const client = core.createTelemetryStore(); + const result = await client.ingestEnvelope(spans("one")); + expect(result).toMatchObject({ + status: "processed", + envelopeId: "one", + diagnostics: [], + }); + expect(client.getDatabaseQuerySummaries({ traceId: "trace" })).toMatchObject([ + { count: 1, totalDurationMs: 125, averageDurationMs: 125 }, + ]); + expect(client.getTrace("trace")?.spans.size).toBe(1); + expect(client.getSpans({ traceId: "absent" })).toEqual([]); + expect(client.getDatabaseQuerySummaries({ traceId: "absent" })).toEqual([]); + expect(client).not.toHaveProperty("store"); + client.dispose(); +}); + +test("subscribers see exactly one coherent update per ingestion", async () => { + const client = core.createTelemetryStore(); + const seen: number[] = []; + client.subscribe((snapshot) => { + expect(snapshot.events).toHaveLength(3); + expect(snapshot.traces).toHaveLength(1); + expect(client.getDatabaseQuerySummaries()[0].count).toBe(3); + seen.push(snapshot.revision); + }); + const initial = client.getSnapshot(); + await client.ingestEnvelope(spans("three", "trace", 3)); + expect(seen).toHaveLength(1); + expect(client.getSnapshot()).toBe(client.getSnapshot()); + expect(initial.events).toEqual([]); + client.dispose(); +}); + +test("returned events, nested maps and earlier snapshots cannot mutate engine state", async () => { + const client = core.createTelemetryStore(); + await client.ingestEnvelope(spans("one")); + const snapshot = client.getSnapshot(); + const trace = client.getTrace("trace")!; + expect(() => (trace.spans as Map).clear()).toThrow(); + expect(() => { + (trace.spans.values().next().value as { description: string }).description = + "changed"; + }).toThrow(); + expect(() => (client.getEvents() as unknown[]).pop()).toThrow(); + trace.spans.forEach((_value, _key, map) => + expect(() => (map as Map).clear()).toThrow(), + ); + await client.ingestEnvelope(spans("two")); + expect(snapshot.traces[0].spans.size).toBe(1); + expect(client.getTrace("trace")?.spans.size).toBe(2); + expect(client.getDatabaseQuerySummaries()[0].description).toBe("SELECT 1"); + client.dispose(); +}); + +test("snapshots preserve own prototype-named JSON fields without inheriting their values", async () => { + const client = core.createTelemetryStore(); + const input = JSON.parse( + '[{"__spotlight_envelope_id":"proto"},[[{"type":"event"},{"event_id":"proto","timestamp":1,"extra":{"__proto__":{"unexpected":true},"kept":1}}]]]', + ); + expect(await client.ingestEnvelope(input)).toMatchObject({ + status: "processed", + }); + const extra = client.getEvent("proto")!.extra!; + expect(Object.hasOwn(extra, "__proto__")).toBe(true); + expect(extra.unexpected).toBeUndefined(); + expect(JSON.stringify(client.getEnvelope("proto"))).toBe( + JSON.stringify(input), + ); + expect(() => { + (extra.__proto__ as { unexpected: boolean }).unexpected = false; + }).toThrow(); + client.dispose(); +}); + +test("invalid and unsupported envelopes return diagnostics without changing the snapshot", async () => { + const client = core.createTelemetryStore(); + const initial = client.getSnapshot(); + expect(await client.ingestEnvelope(null)).toMatchObject({ + status: "invalid", + diagnostics: [{ code: "invalid_envelope" }], + }); + expect(client.getSnapshot()).toBe(initial); + expect( + await client.ingestEnvelope([{}, [[{ type: "future" }, {}]]]), + ).toMatchObject({ + status: "ignored", + reason: "unsupported_items", + }); + expect(await client.ingestEnvelope(spans("duplicate"))).toMatchObject({ + status: "processed", + }); + const snapshot = client.getSnapshot(); + expect(await client.ingestEnvelope(spans("duplicate"))).toMatchObject({ + status: "ignored", + reason: "duplicate", + }); + expect(client.getSnapshot()).toBe(snapshot); + client.dispose(); +}); + +test("invalid parsed input reports the same diagnostic to the observer and caller", async () => { + const diagnostics: string[] = []; + const client = core.createTelemetryStore({ + onDiagnostic: (diagnostic) => diagnostics.push(diagnostic.code), + }); + const result = await client.ingestEnvelope(null); + expect(diagnostics).toEqual(["invalid_envelope"]); + expect(result.diagnostics.map((diagnostic) => diagnostic.code)).toEqual( + diagnostics, + ); + client.dispose(); +}); + +test("a rejected envelope waits for sibling enrichment and cannot poison later events", async () => { + let release!: () => void; + let entered!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const ready = new Promise((resolve) => { + entered = resolve; + }); + let enrichments = 0; + const client = core.createTelemetryStore({ + enrichStacktrace: async (event) => { + enrichments++; + if (enrichments === 1) { + entered(); + await gate; + } + event.exception.values![0].value = + enrichments === 1 ? "invalid-cached" : "valid-context"; + }, + }); + const error = { + event_id: "same", + timestamp: 1, + exception: { values: [{ type: "Error", value: "original" }] }, + }; + const pending = client.ingestEnvelope([ + {}, + [ + [{ type: "event" }, error], + [{ type: "log" }, { items: [null] }], + ], + ]); + await ready; + let finished = false; + void pending.then(() => { + finished = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(finished).toBe(false); + release(); + expect(await pending).toMatchObject({ status: "invalid" }); + expect(client.getEvents()).toEqual([]); + await client.ingestEnvelope([{}, [[{ type: "event" }, error]]]); + expect(enrichments).toBe(2); + expect(client.getErrors()[0].exception.values?.[0].value).toBe( + "valid-context", + ); + client.dispose(); +}); + +test("malformed spans have diagnostics while mixed valid spans still process", async () => { + const client = core.createTelemetryStore(); + expect( + await client.ingestEnvelope([ + {}, + [[{ type: "span" }, { version: 2, items: [{}] }]], + ]), + ).toMatchObject({ + status: "invalid", + diagnostics: [{ code: "invalid_span" }], + }); + const input = spans("partial"); + (input[1][0][1] as { items: unknown[] }).items.push({}); + expect(await client.ingestEnvelope(input)).toMatchObject({ + status: "processed", + diagnostics: [{ code: "invalid_span" }], + }); + expect(client.getSpans()).toHaveLength(1); + client.dispose(); +}); + +test("retention preserves identities of profile spans derived from retained envelopes", async () => { + const { readFileSync } = await import("node:fs"); + const { createTelemetryPipeline } = await import("../sdk/node.ts"); + const client = createTelemetryPipeline({ retention: { maxEnvelopes: 3 } }); + await client.ingestEnvelope(spans("evict-me")); + for (const file of ["transaction.txt", "profile_chunk_1.txt"]) { + const body = readFileSync( + new URL(`../../_fixtures/continuous_profiling/${file}`, import.meta.url), + ); + const result = await client.ingestPayload({ body }); + expect(result.status).toBe("processed"); + } + const traceId = "f1e2d3c4b5a6978012345678901234ab"; + const ids = client.getSpans({ traceId }).map((span) => span.span_id); + expect( + client + .getSpans({ traceId }) + .some((span) => span.tags?.source === "profile"), + ).toBe(true); + await client.ingestEnvelope(spans("new")); + expect(client.getSpans({ traceId }).map((span) => span.span_id)).toEqual(ids); + client.dispose(); +}); + +test("count retention evicts related events, traces, logs and profiles coherently", async () => { + const client = core.createTelemetryStore({ retention: { maxEnvelopes: 1 } }); + await client.ingestEnvelope([ + { __spotlight_envelope_id: "old" }, + [ + [ + { type: "log" }, + { + items: [ + { body: "old", timestamp: 1, trace_id: "old", attributes: {} }, + ], + }, + ], + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "old", + span_id: "old", + name: "SELECT old", + start_timestamp: 1, + end_timestamp: 2, + attributes: { "sentry.op": { value: "db" } }, + }, + ], + }, + ], + ], + ]); + await client.ingestEnvelope(spans("new", "new")); + expect(client.getEnvelopes()).toHaveLength(1); + expect(client.getTrace("old")).toBeUndefined(); + expect(client.getLogs({ traceId: "old" })).toEqual([]); + expect(client.getEvents()).toHaveLength(1); + expect(client.getDatabaseQuerySummaries()).toMatchObject([ + { description: "SELECT 1", count: 1 }, + ]); + client.dispose(); +}); + +test("retention replay preserves generated log IDs and does not repeat enrichment", async () => { + let enrichments = 0; + const client = core.createTelemetryStore({ + retention: { maxEnvelopes: 2 }, + enrichStacktrace: async () => { + enrichments++; + }, + }); + await client.ingestEnvelope(spans("first")); + await client.ingestEnvelope([ + { __spotlight_envelope_id: "retained" }, + [ + [ + { type: "log" }, + { items: [{ body: "hello", timestamp: 1, attributes: {} }] }, + ], + [ + { type: "event" }, + { + event_id: "error", + timestamp: 1, + exception: { values: [{ type: "Error", value: "error" }] }, + }, + ], + ], + ]); + const id = client.getLogs()[0].id; + await client.ingestEnvelope(spans("last")); + expect(client.getLogs()[0].id).toBe(id); + expect(enrichments).toBe(1); + expect(client.getErrors()).toHaveLength(1); + client.dispose(); +}); + +test("duplicate error IDs have one deterministic enrichment owner across replay", async () => { + let release!: () => void; + let entered!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const ready = new Promise((resolve) => { + entered = resolve; + }); + let calls = 0; + const client = core.createTelemetryStore({ + retention: { maxEnvelopes: 2 }, + enrichStacktrace: async () => { + calls++; + if (calls === 1) { + entered(); + await gate; + } + }, + }); + await client.ingestEnvelope(spans("old")); + const pending = client.ingestEnvelope([ + { __spotlight_envelope_id: "keep" }, + ["first", "second"].map((value) => [ + { type: "event" }, + { + event_id: "same", + timestamp: 1, + exception: { values: [{ type: "Error", value }] }, + }, + ]), + ]); + await ready; + release(); + await pending; + expect(calls).toBe(1); + expect(client.getErrors()[0].exception.values?.[0].value).toBe("first"); + await client.ingestEnvelope(spans("new")); + expect(client.getErrors()[0].exception.values?.[0].value).toBe("first"); + expect(calls).toBe(1); + client.dispose(); +}); + +test("evicting an error owner enriches the retained duplicate from its own exception", async () => { + const seen: string[] = []; + const client = core.createTelemetryStore({ + retention: { maxEnvelopes: 3 }, + enrichStacktrace: async (event) => { + const exception = event.exception.values![0]; + seen.push(exception.value!); + exception.value = `enriched:${exception.value}`; + }, + }); + for (const id of ["old", "retained"]) { + await client.ingestEnvelope([ + { __spotlight_envelope_id: id }, + [ + [ + { type: "event" }, + { + event_id: "same", + timestamp: 1, + exception: { values: [{ value: id }] }, + }, + ], + ], + ]); + if (id === "old") await client.ingestEnvelope(spans("middle")); + } + expect(client.getErrors()[0].exception.values![0].value).toBe("enriched:old"); + await client.ingestEnvelope(spans("new")); + expect(client.getErrors()[0].exception.values![0].value).toBe( + "enriched:retained", + ); + expect(seen).toEqual(["old", "retained"]); + await client.ingestEnvelope(spans("next")); + expect(seen).toEqual(["old", "retained"]); + client.dispose(); +}); + +test("SDK last-seen times do not advance when retained envelopes replay", async () => { + vi.useFakeTimers(); + const client = core.createTelemetryStore({ retention: { maxEnvelopes: 2 } }); + for (const [index, name] of ["old", "keep", "new"].entries()) { + vi.setSystemTime(1000 * (index + 1)); + await client.ingestEnvelope([ + { __spotlight_envelope_id: name }, + [ + [ + { type: "event" }, + { event_id: name, timestamp: 1, sdk: { name, version: "1" } }, + ], + ], + ]); + } + expect(client.getSdks().find((sdk) => sdk.name === "keep")?.lastSeen).toBe( + 2000, + ); + client.dispose(); +}); + +test("age retention expires while idle and disposes its timer", async () => { + vi.useFakeTimers(); + const client = core.createTelemetryStore({ retention: { maxAgeMs: 1000 } }); + await client.ingestEnvelope(spans("old")); + await vi.advanceTimersByTimeAsync(1000); + expect(client.getTraces()).toEqual([]); + expect(client.getEnvelopes()).toEqual([]); + expect(vi.getTimerCount()).toBe(0); + await client.ingestEnvelope(spans("new")); + client.dispose(); + expect(vi.getTimerCount()).toBe(0); + expect(await client.ingestEnvelope(spans("closed"))).toMatchObject({ + status: "ignored", + reason: "disposed", + }); +}); + +test("reset invalidates active and queued work while allowing immediate new ingestion", async () => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let entered!: () => void; + const ready = new Promise((resolve) => { + entered = resolve; + }); + const client = core.createTelemetryStore({ + enrichStacktrace: () => { + entered(); + return gate; + }, + }); + const active = client.ingestEnvelope([ + { __spotlight_envelope_id: "error" }, + [ + [ + { type: "event" }, + { event_id: "error", timestamp: 1, exception: { values: [] } }, + ], + ], + ]); + await ready; + const queued = client.ingestEnvelope(spans("queued")); + client.reset(); + await client.ingestEnvelope(spans("new")); + expect(client.getEvents()).toHaveLength(1); + release(); + expect(await active).toMatchObject({ status: "ignored", reason: "reset" }); + expect(await queued).toMatchObject({ status: "ignored", reason: "reset" }); + expect(client.getEvents()).toHaveLength(1); + expect(client.getErrors()).toEqual([]); + client.dispose(); +}); + +test("subscribers can reset without a later notification restoring stale state", async () => { + const client = core.createTelemetryStore(); + let reset = false; + const observed: number[] = []; + client.subscribe((snapshot) => { + if (!reset && snapshot.events.length) { + reset = true; + client.reset(); + } + }); + client.subscribe((snapshot) => observed.push(snapshot.events.length)); + await client.ingestEnvelope(spans("old")); + expect(client.getTraces()).toEqual([]); + expect(observed).toEqual([0]); + client.dispose(); +}); diff --git a/packages/telemetry/src/telemetry/constants/performance.ts b/packages/telemetry/src/telemetry/constants/performance.ts new file mode 100644 index 000000000..c020eaf84 --- /dev/null +++ b/packages/telemetry/src/telemetry/constants/performance.ts @@ -0,0 +1,94 @@ +export const PERFORMANCE_SCORE_PROFILES = { + profiles: [ + { + name: "Chrome", + scoreComponents: [ + { + measurement: "fcp", + weight: 0.15, + p10: 900.0, + p50: 1600.0, + optional: false, + }, + { + measurement: "lcp", + weight: 0.3, + p10: 1200.0, + p50: 2400.0, + optional: false, + }, + { + measurement: "fid", + weight: 0.3, + p10: 100.0, + p50: 300.0, + optional: true, + }, + { + measurement: "cls", + weight: 0.15, + p10: 0.1, + p50: 0.25, + optional: false, + }, + { + measurement: "ttfb", + weight: 0.1, + p10: 200.0, + p50: 400.0, + optional: false, + }, + ], + condition: { + op: "eq", + name: "event.contexts.browser.name", + value: "Chrome", + }, + }, + // TODO: Currently not getting browser data in events, so made these comments and sticked to chrome always. + // { + // "name": "Firefox", + // "scoreComponents": [ + // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, + // {"measurement": "lcp", "weight": 0.0, "p10": 1200.0, "p50": 2400.0, "optional": false}, + // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, + // {"measurement": "cls", "weight": 0.0, "p10": 0.1, "p50": 0.25, "optional": false}, + // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} + // ], + // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Firefox"} + // }, + // { + // "name": "Safari", + // "scoreComponents": [ + // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, + // {"measurement": "lcp", "weight": 0.0, "p10": 1200.0, "p50": 2400.0, "optional": false}, + // {"measurement": "fid", "weight": 0.0, "p10": 100.0, "p50": 300.0, "optional": true}, + // {"measurement": "cls", "weight": 0.0, "p10": 0.1, "p50": 0.25, "optional": false}, + // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} + // ], + // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Safari"} + // }, + // { + // "name": "Edge", + // "scoreComponents": [ + // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, + // {"measurement": "lcp", "weight": 0.30, "p10": 1200.0, "p50": 2400.0, "optional": false}, + // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, + // {"measurement": "cls", "weight": 0.15, "p10": 0.1, "p50": 0.25, "optional": false}, + // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} + // ], + // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Edge"} + // }, + // { + // "name": "Opera", + // "scoreComponents": [ + // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, + // {"measurement": "lcp", "weight": 0.30, "p10": 1200.0, "p50": 2400.0, "optional": false}, + // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, + // {"measurement": "cls", "weight": 0.15, "p10": 0.1, "p50": 0.25, "optional": false}, + // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} + // ], + // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Opera"} + // } + ], +}; diff --git a/packages/telemetry/src/telemetry/constants/sentry.ts b/packages/telemetry/src/telemetry/constants/sentry.ts new file mode 100644 index 000000000..1bedc69f2 --- /dev/null +++ b/packages/telemetry/src/telemetry/constants/sentry.ts @@ -0,0 +1,14 @@ +export const RAW_TYPES = new Set(["attachment", "replay_video", "statsd"]); +export const ERROR_EVENT_TYPES = new Set(["event", "error"]); +export const TRACE_EVENT_TYPES = new Set(["transaction"]); +export const PROFILE_EVENT_TYPES = new Set(["profile", "profile_chunk"]); +export const LOG_EVENT_TYPES = new Set(["log"]); +export const FEEDBACK_EVENT_TYPES = new Set(["feedback"]); +export const ATTACHMENT_EVENT_TYPES = new Set(["attachment"]); +export const SUPPORTED_EVENT_TYPES = new Set([ + ...ERROR_EVENT_TYPES, + ...TRACE_EVENT_TYPES, + ...PROFILE_EVENT_TYPES, + ...LOG_EVENT_TYPES, + ...FEEDBACK_EVENT_TYPES, +]); diff --git a/packages/telemetry/src/telemetry/contracts.ts b/packages/telemetry/src/telemetry/contracts.ts new file mode 100644 index 000000000..976fe5a87 --- /dev/null +++ b/packages/telemetry/src/telemetry/contracts.ts @@ -0,0 +1,87 @@ +/** Machine-readable ingestion outcomes. Messages are explanatory, never control flow. */ +export type DiagnosticCode = + | "invalid_envelope" + | "invalid_span" + | "invalid_body" + | "invalid_content_type" + | "unsupported_encoding" + | "decompression_failed" + | "payload_too_large" + | "decoded_payload_too_large" + | "invalid_delivery_id" + | "enrichment_failed" + | "enrichment_timeout" + | "subscriber_failed" + | "unsupported_version"; +export type Diagnostic = { + readonly code: DiagnosticCode; + readonly message: string; + readonly severity: "warning" | "error"; + readonly itemIndex?: number; + readonly spanIndex?: number; +}; +export type IgnoreReason = + | "duplicate" + | "unsupported_items" + | "unsupported_content_type" + | "reset" + | "disposed" + | "aborted" + | "queue_full"; +export type IngestCounts = { + readonly events: number; + readonly spans: number; + readonly logs: number; + readonly skippedItems: number; + readonly duplicateEvents: number; +}; +export type ReceiptMetadata = { + readonly revision: number; + readonly retained: boolean; + readonly counts: IngestCounts; +}; +export type IngestOutcome = + | { + readonly status: "processed"; + readonly envelopeId: string; + readonly diagnostics?: readonly Diagnostic[]; + } + | { + readonly status: "ignored"; + readonly reason: IgnoreReason; + readonly envelopeId?: string; + readonly diagnostics?: readonly Diagnostic[]; + } + | { readonly status: "invalid"; readonly diagnostics: readonly Diagnostic[] }; +export type IngestResult = ReceiptMetadata & + ( + | { + readonly status: "processed"; + readonly envelopeId: string; + readonly diagnostics: readonly Diagnostic[]; + } + | { + readonly status: "ignored"; + readonly reason: IgnoreReason; + readonly envelopeId?: string; + readonly diagnostics: readonly Diagnostic[]; + } + | { + readonly status: "invalid"; + readonly diagnostics: readonly Diagnostic[]; + } + ); +export type IngestOptions = { deliveryId?: string; signal?: AbortSignal }; +export type BatchOptions = Pick; +export type ChangeMetadata = { + readonly type: "ingest" | "evict" | "reset" | "dispose"; + readonly revision: number; + readonly envelopeIds: readonly string[]; +}; +export type IngestStats = { + readonly pending: number; + readonly retainedEnvelopes: number; + readonly retainedBytes: number; + readonly revision: number; + readonly disposed: boolean; +}; diff --git a/packages/telemetry/src/telemetry/data/profiles.ts b/packages/telemetry/src/telemetry/data/profiles.ts new file mode 100644 index 000000000..1104886d0 --- /dev/null +++ b/packages/telemetry/src/telemetry/data/profiles.ts @@ -0,0 +1,265 @@ +import { log } from "../logger.ts"; +import type { SentryProfileWithTraceMeta } from "../store/types.ts"; +import type { Span, Trace } from "../types.ts"; +import { getFunctionNameFromFrame } from "../utils/profileUtils.ts"; +import { compareSpans } from "../utils/traces.ts"; + +/** Non-cryptographic FNV-1a identity for a derived profile frame, stable across replays. */ +function profileSpanId(parts: unknown[]): string { + let hash = 0x6c62272e07bb014262b821756295c58dn; + for (const byte of new TextEncoder().encode(JSON.stringify(parts))) { + hash = BigInt.asUintN( + 128, + (hash ^ BigInt(byte)) * 0x1000000000000000000013bn, + ); + } + return hash.toString(16).padStart(32, "0"); +} + +/** + * Groups consequent spans with the same description and op into a single span per each level. + * Essentially a BFS traversal of the spans tree. + * @param spans Span[] A list of spans to consolidate, sorted by their start_timestamp + * @returns Span[] A list of spans with the same description and op consolidated into a single span + */ +function consolidateSpans(trace: Trace, spans: Span[]): Span[] { + const consolidatedSpans: Span[] = []; + let lastSpan = spans[0]; + let spanIdx = 1; + while (spanIdx < spans.length + 1) { + const span = spans[spanIdx]; + spanIdx += 1; + if ( + span && + span.description === lastSpan.description && + span.op === lastSpan.op + ) { + // Require the spans are sorted by start_timestamp + lastSpan.timestamp = span.timestamp; + if (span.children) { + if (lastSpan.children) { + for (const child of span.children) { + lastSpan.children.push(child); + } + lastSpan.children.sort(compareSpans); + } else { + lastSpan.children = span.children; + } + } + } else { + // Discard very short spans + if (lastSpan.timestamp - lastSpan.start_timestamp > 0) { + if (lastSpan.children) { + lastSpan.children = consolidateSpans(trace, lastSpan.children); + } + consolidatedSpans.push(lastSpan); + trace.spans.set(lastSpan.span_id, lastSpan); + } + lastSpan = span; + } + } + + return consolidatedSpans; +} + +// This is done per platform right now but we may want to make it use +// SDK or something more specific, especially for JS as `javascript` may +// mean browser, node, etc. +const SENTRY_FRAME_FILTER_PER_PLATFORM: Record< + string, + ( + this: SentryProfileWithTraceMeta["frames"], + frameIdx: number, + ) => boolean | undefined +> = { + python: function (frameIdx) { + return this[frameIdx].module?.startsWith("sentry_sdk."); + }, + javascript: function (frameIdx) { + const frame = this[frameIdx]; + const module = frame.module; + if (module) { + return ( + module.startsWith("@sentry") || + module.startsWith("@opentelemetry.instrumentation") + ); + } + // This one below is to match things like `http://localhost:3000/node_modules/.vite/deps/@sentry_react.js?v=6942e78f` etc. + return frame.abs_path + ? /\/node_modules\/.*\/@(sentry|opentelemetry)[^a-z0-9]/.test( + frame.abs_path, + ) + : false; + }, +}; + +export function getSpansFromProfile( + trace: Trace, + profile: SentryProfileWithTraceMeta, + parent_span_id: string | undefined, + startTs: number, + endTs: number, + threadIds: Set, +): Span[] { + threadIds.add(profile.active_thread_id); + + const sentryFrameFilter = + profile.platform && SENTRY_FRAME_FILTER_PER_PLATFORM[profile.platform]; + // Try to fill in the gaps from profile data + const fillerSpans: Span[] = []; + for (let sampleIdx = 0; sampleIdx < profile.samples.length; sampleIdx++) { + const sample = profile.samples[sampleIdx]; + if (sample.thread_id && !threadIds.has(sample.thread_id)) { + continue; + } + const sampleTs = sample.start_timestamp; + if (sampleTs < startTs || sampleTs > endTs) { + continue; + } + const nextSample = profile.samples[sampleIdx + 1]; + const timestamp = nextSample ? nextSample.start_timestamp : endTs; + + if (timestamp > endTs) { + continue; + } + const commonAttributes = { + start_timestamp: sampleTs, + timestamp, + trace_id: trace.trace_id, + status: "ok", + tags: { source: "profile" }, + data: { + "thread.id": sample.thread_id, + "thread.name": + profile.thread_metadata?.[ + sample.thread_id as keyof typeof profile.thread_metadata + ]?.name, + }, + }; + const identity = [ + trace.trace_id, + profile.timestamp, + parent_span_id, + startTs, + endTs, + sampleIdx, + sample.thread_id, + ]; + const sampleSpan: Span = { + span_id: profileSpanId([...identity, "thread"]), + parent_span_id, + ...commonAttributes, + op: "Thread", + description: + profile.thread_metadata?.[ + sample.thread_id as keyof typeof profile.thread_metadata + ]?.name || `Thread ${sample.thread_id}`, + data: { + thread_id: sample.thread_id, + }, + }; + let currentSpan = sampleSpan; + const currentStack = profile.stacks[sample.stack_id]; + const lastSentryFrameIdx = sentryFrameFilter + ? currentStack.findLastIndex(sentryFrameFilter, profile.frames) + : 0; + for ( + let frameIdxIdx = lastSentryFrameIdx + 1; + frameIdxIdx < currentStack.length; + frameIdxIdx++ + ) { + const frame = profile.frames[currentStack[frameIdxIdx]]; + // XXX: We may wanna skip frames that doesn't have `in_app` set to true + // that said it's better to have this as a dynamic filter + const [description, op] = getFunctionNameFromFrame(frame).split("@"); + const spanFromFrame: Span = { + span_id: profileSpanId([...identity, "frame", frameIdxIdx]), + parent_span_id: currentSpan.span_id, + ...commonAttributes, + op, + description, + data: { + ...frame, + }, + }; + currentSpan.children = [spanFromFrame]; + currentSpan = spanFromFrame; + } + fillerSpans.push(sampleSpan); + } + + if (!fillerSpans.length) { + return []; + } + const consolidated = consolidateSpans(trace, fillerSpans); + // Remove the extra layer of nesting if there is only one span which should be the "Thread" span + return ( + consolidated.length === 1 ? consolidated[0].children || [] : consolidated + ).filter( + (span) => + span.timestamp - span.start_timestamp > 0 && span.timestamp <= endTs, + ); +} + +/** + * Modifies the spanTree in place recursively by adding spans from the + * profile data where there are gaps in the trace data. + * @param spanTree Span[] The tree of spans to graft profile spans into + */ +export function graftProfileSpans( + trace: Trace, + profile: SentryProfileWithTraceMeta, + spanTree: Span[] = trace.spanTree, + parent: Span | Trace = trace, +) { + log(`Grafting profile spans into trace ${trace.trace_id}`); + if (trace.profileGrafted) { + log(`Trace already has profile grafted ${trace.trace_id}`); + return; + } + + let idx = -1; + while (idx < spanTree.length) { + const span = spanTree[idx] as Span | undefined; + if (span?.tags?.source === "profile") { + idx += 1; + continue; + } + const nextSpan = spanTree[idx + 1]; + if (nextSpan?.tags?.source === "profile") { + idx += 1; + continue; + } + + const startTs = span ? span.timestamp : parent.start_timestamp; + const endTs = nextSpan ? nextSpan.start_timestamp : parent.timestamp; + const threadIds = new Set([ + span?.data?.threadId, + nextSpan?.data?.threadId, + parent?.data?.threadId, + ]); + threadIds.delete(undefined); + if (endTs - startTs > 0) { + const fillers = getSpansFromProfile( + trace, + profile, + parent.span_id, + startTs, + endTs, + threadIds as Set, + ); + if (fillers.length) { + spanTree.splice(idx + 1, 0, ...fillers); + idx += fillers.length; + } + } + if (span) { + span.children ??= []; + graftProfileSpans(trace, profile, span.children, span); + } + idx += 1; + } + // Only mark as grafted at the top level to avoid early quitting during + // recursive calls above for child spans + trace.profileGrafted = trace.spanTree === spanTree; +} diff --git a/packages/telemetry/src/telemetry/envelope/codec.spec.ts b/packages/telemetry/src/telemetry/envelope/codec.spec.ts new file mode 100644 index 000000000..0a76814db --- /dev/null +++ b/packages/telemetry/src/telemetry/envelope/codec.spec.ts @@ -0,0 +1,103 @@ +import type { Envelope } from "@sentry/core"; +import { describe, expect, test, vi } from "vitest"; +import { normalizeEnvelope } from "../internal/envelopes.ts"; +import { decodeEnvelope } from "./codec.ts"; + +const encode = (value: string) => new TextEncoder().encode(value); + +describe("portable byte envelope codec", () => { + test("parses Uint8Array text attachment bytes with explicit byte lengths", () => { + const text = "é\nhello"; + const payload = encode(text); + const header = encode( + `{}\n${JSON.stringify({ type: "attachment", content_type: "text/plain", length: payload.length })}\n`, + ); + const bytes = new Uint8Array(header.length + payload.length); + bytes.set(header); + bytes.set(payload, header.length); + const envelope = decodeEnvelope(bytes, undefined, undefined, true)!; + expect(envelope[1][0][1]).toEqual({ data: text }); + expect(envelope[0].__spotlight_inferred_source).toBe("server"); + }); + + test("uses the same text and binary policies for parsed and wire envelopes", () => { + for (const content_type of [ + "text/plain", + "application/json", + "application/octet-stream", + undefined, + ]) { + const payload = encode("\ufeffhéllo"); + const itemHeader = { + type: "attachment", + content_type, + length: payload.length, + }; + const wireHeader = encode(`{}\n${JSON.stringify(itemHeader)}\n`); + const wire = new Uint8Array(wireHeader.length + payload.length); + wire.set(wireHeader); + wire.set(payload, wireHeader.length); + const decoded = decodeEnvelope(wire, undefined, undefined, true)!; + const parsed = normalizeEnvelope([ + {}, + [[itemHeader, payload]], + ] as Envelope); + expect(decoded[1]).toEqual(parsed[1]); + } + }); + + test("retains the legacy statsd default text policy", () => { + const wire = decodeEnvelope( + encode('{}\n{"type":"statsd"}\ncount:1|c'), + undefined, + undefined, + true, + )!; + const parsed = normalizeEnvelope([ + {}, + [[{ type: "statsd" }, "count:1|c"]], + ] as unknown as Envelope); + expect(wire[1]).toEqual(parsed[1]); + expect(wire[1][0][1]).toEqual({ data: "count:1|c" }); + }); + + test("preserves permissive truncation handling while strict decoding rejects it", () => { + const bytes = encode('{}\n{"type":"attachment","length":30}\nshort'); + const errors = vi.fn(); + expect(decodeEnvelope(bytes, undefined, errors)).not.toBeNull(); + expect(errors).not.toHaveBeenCalled(); + expect(decodeEnvelope(bytes, undefined, errors, true)).toBeNull(); + expect(errors).toHaveBeenCalledOnce(); + }); + + test.each([-1, -100, 1.5, "1", null, false])( + "rejects invalid item length %j in both decoder modes", + (length) => { + const bytes = encode( + `{}\n${JSON.stringify({ type: "attachment", length })}\n`, + ); + for (const strict of [false, true]) { + const errors = vi.fn(); + expect(decodeEnvelope(bytes, undefined, errors, strict)).toBeNull(); + expect(errors).toHaveBeenCalledOnce(); + } + }, + ); + + test.each([ + [ + "sentry.javascript.nextjs", + "Mozilla/5.0 Chrome/131.0.0.0 Safari/537.36", + "browser", + ], + ["sentry.cocoa", "Mozilla/5.0 Safari/605", "mobile"], + ["sentry.javascript.node", "Node.js/24", "server"], + ])("preserves source inference for %s", (name, userAgent, source) => { + const envelope = decodeEnvelope( + encode(`${JSON.stringify({ sdk: { name, version: "1" } })}\n`), + userAgent, + )!; + expect(envelope[0].__spotlight_sender_user_agent).toBe(userAgent); + expect(envelope[0].__spotlight_inferred_source).toBe(source); + }); +}); diff --git a/packages/telemetry/src/telemetry/envelope/codec.ts b/packages/telemetry/src/telemetry/envelope/codec.ts new file mode 100644 index 000000000..673c558c2 --- /dev/null +++ b/packages/telemetry/src/telemetry/envelope/codec.ts @@ -0,0 +1,171 @@ +import type { Envelope, EnvelopeItem } from "@sentry/core"; +import { type UUID, uuidv7obj } from "uuidv7"; +import { RAW_TYPES, SUPPORTED_EVENT_TYPES } from "../constants/sentry.ts"; +import { opaquePayload } from "./payload.ts"; +import { type SourceType, inferEnvelopeSource } from "./source.ts"; + +/** + * Spotlight-specific extensions added to envelope headers for internal tracking + */ +export type SpotlightEnvelopeExtensions = { + __spotlight_envelope_id: UUID; + __spotlight_sender_user_agent?: string; + __spotlight_inferred_source?: SourceType; +}; + +export type DecodedEnvelope = [ + Envelope[0] & SpotlightEnvelopeExtensions, + Envelope[1], +]; + +/** + * Implements parser for + * @see https://develop.sentry.dev/sdk/envelopes/#serialization-format + * @param data Serialized UTF-8 envelope bytes + * @param senderUserAgent Optional User-Agent header from the HTTP request sending the envelope + * @param reportError Optional error observer; silent by default + * @param strict Reject truncated payloads and invalid explicit lengths + * @returns parsed envelope + */ +export function decodeEnvelope( + data: Uint8Array, + senderUserAgent?: string, + reportError: (error: unknown) => void = () => {}, + strict = false, +): DecodedEnvelope | null { + let buffer = data; + + function readLine(length?: number) { + const cursor = length ?? getLineEnd(buffer); + const line = buffer.subarray(0, cursor); + buffer = buffer.subarray(cursor + 1); + return line; + } + + const envelopeHeader = parseJSONFromBuffer( + readLine(), + reportError, + ) as Envelope[0]; + if ( + !envelopeHeader || + typeof envelopeHeader !== "object" || + Array.isArray(envelopeHeader) + ) { + reportError("Malformed envelope header, skipping..."); + return null; + } + + envelopeHeader.__spotlight_envelope_id = uuidv7obj(); + + // Store sender User-Agent for improved SDK categorization + if (senderUserAgent) { + envelopeHeader.__spotlight_sender_user_agent = senderUserAgent; + } + + const items: EnvelopeItem[] = []; + while (buffer.length) { + if (strict && buffer.indexOf(0xa) === -1) { + reportError("Truncated envelope item header, skipping envelope..."); + return null; + } + const itemHeader = parseJSONFromBuffer( + readLine(), + reportError, + ) as EnvelopeItem[0]; + if ( + !itemHeader || + typeof itemHeader !== "object" || + Array.isArray(itemHeader) + ) { + reportError("Malformed envelope item header, skipping envelope..."); + return null; + } + const payloadLength = itemHeader.length; + // Permissive mode allows truncated payloads, never invalid buffer indices. + if ( + payloadLength !== undefined && + (!Number.isSafeInteger(payloadLength) || payloadLength < 0) + ) { + reportError("Malformed envelope item length, skipping envelope..."); + return null; + } + if ( + strict && + payloadLength !== undefined && + (payloadLength > buffer.length || + (payloadLength < buffer.length && buffer[payloadLength] !== 0xa)) + ) { + reportError("Malformed envelope item length, skipping envelope..."); + return null; + } + const itemPayloadRaw = readLine(payloadLength); + + let itemPayload: EnvelopeItem[1]; + try { + if ( + RAW_TYPES.has(itemHeader.type) || + (strict && + !SUPPORTED_EVENT_TYPES.has(itemHeader.type) && + itemHeader.type !== "span") + ) { + if ( + !itemHeader.content_type && + // @ts-expect-error -- statsd is an old and deprecated event type but have envelopes for that + itemHeader.type === "statsd" + ) { + // @ts-expect-error -- same as above + itemHeader.content_type = "text/plain"; + } + itemPayload = opaquePayload( + itemPayloadRaw, + itemHeader.content_type as string, + ) as EnvelopeItem[1]; + } else { + itemPayload = parseJSONFromBuffer(itemPayloadRaw, reportError); + if (!itemPayload) { + reportError(itemHeader); + } else if (itemHeader.type) { + // data sanitization + // @ts-expect-error ts(2339) -- We should really stop adding type to payloads + itemPayload.type = itemHeader.type; + } + } + } catch (err) { + itemPayload = itemPayloadRaw; + reportError(err); + } + + items.push([itemHeader, itemPayload] as EnvelopeItem); + } + + // Infer the envelope source (browser, server, or mobile) for UI display + // Scan all events to find a deterministic match + const eventPayloads = items.map(([, payload]) => payload); + envelopeHeader.__spotlight_inferred_source = inferEnvelopeSource( + envelopeHeader, + eventPayloads, + ); + + return [envelopeHeader, items] as DecodedEnvelope; +} + +function getLineEnd(data: Uint8Array): number { + let end = data.indexOf(0xa); + if (end === -1) { + end = data.length; + } + + return end; +} + +function parseJSONFromBuffer( + data: Uint8Array, + reportError: (error: unknown) => void, +): T { + try { + return JSON.parse(new TextDecoder().decode(data)) as T; + } catch (err) { + reportError(err); + return null as T; + } +} diff --git a/packages/telemetry/src/telemetry/envelope/payload.ts b/packages/telemetry/src/telemetry/envelope/payload.ts new file mode 100644 index 000000000..5acb56b8a --- /dev/null +++ b/packages/telemetry/src/telemetry/envelope/payload.ts @@ -0,0 +1,30 @@ +/** The same text/binary representation is used for parsed and serialized opaque items. */ +const TEXT_CONTENT_TYPES = new Set([ + "text/plain", + "text/css", + "text/csv", + "text/html", + "text/javascript", + "text/json", + "text/x-json", + "application/json", + "application/ld+json", +]); + +export function opaquePayload( + payload: string | Uint8Array, + contentType?: string, +): { data: string } { + const bytes = + typeof payload === "string" ? new TextEncoder().encode(payload) : payload; + if (TEXT_CONTENT_TYPES.has(contentType ?? "")) { + return { + data: new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes), + }; + } + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return { data: btoa(binary) }; +} diff --git a/packages/telemetry/src/telemetry/envelope/source.ts b/packages/telemetry/src/telemetry/envelope/source.ts new file mode 100644 index 000000000..04b2a9f95 --- /dev/null +++ b/packages/telemetry/src/telemetry/envelope/source.ts @@ -0,0 +1,163 @@ +import type { Envelope } from "@sentry/core"; +import { parseBrowserFromUserAgent } from "./userAgent.ts"; + +export const SOURCE_TYPES = ["browser", "mobile", "server"] as const; +export type SourceType = (typeof SOURCE_TYPES)[number]; + +/** + * Helper to detect if a User-Agent string is from a browser + */ +function isBrowserUserAgent(userAgent: string): boolean { + const parsed = parseBrowserFromUserAgent(userAgent); + return ( + parsed !== "unknown" && + (parsed.includes("Chrome") || + parsed.includes("Firefox") || + parsed.includes("Safari") || + parsed.includes("Edge")) + ); +} + +/** + * Check if a platform string indicates a server environment + */ +function isServerPlatform(platform: string | undefined): boolean { + return ( + platform === "node" || + platform === "python" || + platform === "ruby" || + platform === "php" || + platform === "java" || + platform === "go" || + platform === "rust" || + platform === "perl" || + platform === "elixir" || + platform === "csharp" || + platform === "dotnet" + ); +} + +/** + * Try to infer source from a single event's properties + * Returns the source if deterministically detected, null otherwise + */ +function inferSourceFromEvent(event: any): SourceType | null { + if (!event) return null; + + // Runtime tags check - deterministic browser + if (event.tags?.runtime === "browser") { + return "browser"; + } + + // Runtime context indicates server + if (event.contexts?.runtime?.name) { + return "server"; + } + + // server_name is a server-specific field + if (event.server_name) { + return "server"; + } + + // Platform check + if (isServerPlatform(event.platform)) { + return "server"; + } + + return null; +} + +/** + * Infer the source of an envelope as browser, mobile, or server using multiple signals + * Priority order: + * 1. SDK name (mobile detection) + * 2. Sender User-Agent (from HTTP request header) + * 3. Platform & Runtime tags (from event payloads - scans all events) + * 4. SDK name (browser/server fallback) + * + * Rules based on https://release-registry.services.sentry.io/sdks + * + * @param envelopeHeader The envelope header containing SDK info and spotlight extensions + * @param eventsOrEvent Single event or array of events to scan for source signals (scans until deterministic match) + */ +export function inferEnvelopeSource( + envelopeHeader: Envelope[0], + eventsOrEvent?: unknown, +): SourceType { + // Normalize to array for consistent handling + const events = eventsOrEvent + ? Array.isArray(eventsOrEvent) + ? eventsOrEvent + : [eventsOrEvent] + : []; + const sdkName = envelopeHeader?.sdk?.name || ""; + + // 1. Mobile check (unchanged - already reliable from SDK name) + // Mobile: Native mobile platforms and frameworks + if ( + sdkName.includes("cocoa") || + sdkName.includes("android") || + sdkName.includes("react-native") || + sdkName.includes("flutter") || + sdkName.includes("capacitor") || + sdkName.includes("cordova") || + sdkName.includes("xamarin") || + sdkName.includes("maui") || + sdkName.includes("unity") || + sdkName.includes("kotlin.kmp") + ) { + return "mobile"; + } + + // 2. Sender User-Agent check + const senderUserAgent = (envelopeHeader as any).__spotlight_sender_user_agent; + if (senderUserAgent && typeof senderUserAgent === "string") { + if (isBrowserUserAgent(senderUserAgent)) { + return "browser"; + } + // Server SDKs send server UAs like "Node.js/*", "Python-urllib/*", etc. + // If we have a non-browser UA, we continue to further checks + } + + // 3. Scan all events for deterministic source signals + for (const event of events) { + const source = inferSourceFromEvent(event); + if (source) { + return source; + } + } + + // 4. SDK name check (existing logic as fallback) + // Browser: JavaScript frameworks/libraries (excluding server/native runtimes and meta-frameworks) + if ( + sdkName.startsWith("sentry.javascript.") && + !sdkName.includes("node") && + !sdkName.includes("bun") && + !sdkName.includes("deno") && + !sdkName.includes("electron") && + !sdkName.includes("serverless") && + !sdkName.includes("cloudflare") && + !sdkName.includes("vercel-edge") && + !sdkName.includes("wasm") && + !sdkName.includes("opentelemetry") && + !sdkName.includes("nextjs") && + !sdkName.includes("remix") && + !sdkName.includes("gatsby") && + !sdkName.includes("astro") && + !sdkName.includes("nuxt") && + !sdkName.includes("sveltekit") && + !sdkName.includes("solidstart") && + !sdkName.includes("nestjs") && + !sdkName.includes("tanstackstart") + ) { + return "browser"; + } + + // Special case: Blazor WebAssembly runs in browser + if (sdkName.includes("blazor.webassembly")) { + return "browser"; + } + + // Server: Everything else (node, python, ruby, go, php, java, dotnet, etc.) + return "server"; +} diff --git a/packages/telemetry/src/telemetry/envelope/userAgent.ts b/packages/telemetry/src/telemetry/envelope/userAgent.ts new file mode 100644 index 000000000..5762c5f94 --- /dev/null +++ b/packages/telemetry/src/telemetry/envelope/userAgent.ts @@ -0,0 +1,61 @@ +/** + * Extracts browser name and major version from a User-Agent string + * Returns format like "Chrome/131" or "Firefox/122" + */ +export function parseBrowserFromUserAgent(userAgent: string): string { + if (!userAgent) return "unknown"; + + // Check for Chrome (but not Edge which also contains Chrome) + if (userAgent.includes("Chrome/") && !userAgent.includes("Edg/")) { + const match = userAgent.match(/Chrome\/([\d]+)/); + return match ? `Chrome/${match[1]}` : "Chrome"; + } + + // Check for Firefox + if (userAgent.includes("Firefox/")) { + const match = userAgent.match(/Firefox\/([\d]+)/); + return match ? `Firefox/${match[1]}` : "Firefox"; + } + + // Check for Safari (but not Chrome which also contains Safari) + if (userAgent.includes("Safari/") && !userAgent.includes("Chrome")) { + const match = userAgent.match(/Version\/([\d]+)/); + return match ? `Safari/${match[1]}` : "Safari"; + } + + // Check for Edge + if (userAgent.includes("Edg/")) { + const match = userAgent.match(/Edg\/([\d]+)/); + return match ? `Edge/${match[1]}` : "Edge"; + } + + // Generic fallback - try to extract first meaningful part + const match = userAgent.match(/^([^/\s(]+)/); + return match ? match[1] : "unknown"; +} + +/** + * Builds a client identifier from browser info and optional hostname + * Format: clientName/version (metadata) + * Example: "ui/Chrome/131 (example.com)" + */ +export function buildClientIdentifier( + clientName: string, + userAgent?: string, + hostname?: string, +): string { + let clientId = clientName; + + if (userAgent) { + const browserInfo = parseBrowserFromUserAgent(userAgent); + if (browserInfo !== "unknown") { + clientId = `${clientName}/${browserInfo}`; + } + } + + if (hostname && hostname !== "localhost") { + clientId += ` (${hostname})`; + } + + return clientId; +} diff --git a/packages/telemetry/src/telemetry/index.ts b/packages/telemetry/src/telemetry/index.ts new file mode 100644 index 000000000..b15513e4c --- /dev/null +++ b/packages/telemetry/src/telemetry/index.ts @@ -0,0 +1,730 @@ +import { stringifyValue } from "./utils/stringifyValue.ts"; +import type { Envelope } from "@sentry/core"; +import { SUPPORTED_EVENT_TYPES } from "./constants/sentry.ts"; +import type { + ChangeMetadata, + Diagnostic, + DiagnosticCode, + IgnoreReason, + IngestCounts, + IngestOptions, + IngestResult, +} from "./contracts.ts"; +import { createEngine } from "./internal/engine.ts"; +import { + normalizeEnvelope, + supportsEnvelope, + validateEnvelope, +} from "./internal/envelopes.ts"; +import { readonlyCopy } from "./internal/readonly.ts"; +import { + diagnostic as makeDiagnostic, + receipt, + reportDiagnostic, +} from "./internal/results.ts"; +import { validateRetention } from "./internal/retention.ts"; +import type { + TelemetrySnapshot, + TelemetryStore, + TelemetryStoreOptions, +} from "./public.ts"; +import { createTelemetryReader } from "./reader.ts"; +import { normalizeSpanItem } from "./store/utils/spanProcessor.ts"; +import type { SentryErrorEvent } from "./types.ts"; +import { generateUuidv4 } from "./uuid.ts"; + +type RetainedEnvelope = { + id: string; + envelope: Envelope; + receivedAtMs: number; + bytes: number; +}; +const MiB = 1024 * 1024; +function positive(value: number, name: string) { + if (!Number.isSafeInteger(value) || value < 1) + throw new RangeError(`${name} must be a positive safe integer`); + return value; +} + +/** Portable processing with bounded work, immutable committed reads and explicit receipts. */ +export function createTelemetryStore( + options: TelemetryStoreOptions = {}, +): TelemetryStore { + const retention = validateRetention(options.retention ?? {}); + let maxEnvelopes = retention.maxEnvelopes ?? 500; + let maxBytes = retention.maxBytes ?? 64 * MiB; + const maxPending = positive(options.maxPending ?? 64, "maxPending"); + const maxEnvelopeBytes = positive( + options.maxEnvelopeBytes ?? 32 * MiB, + "maxEnvelopeBytes", + ); + const enrichmentTimeoutMs = positive( + options.enrichmentTimeoutMs ?? 5000, + "enrichmentTimeoutMs", + ); + let maxAgeMs = retention.maxAgeMs; + let generation = 0; + let disposed = false; + let revision = 0; + let pending = 0; + let queue: Promise = Promise.resolve(); + let pruning: Promise | undefined; + let journal: RetainedEnvelope[] = []; + let timer: ReturnType | undefined; + let lifetime = new AbortController(); + const identities = new WeakMap(); + const listeners = new Set< + (snapshot: TelemetrySnapshot, change: ChangeMetadata) => void + >(); + let enrichmentCache = new Map< + string, + { envelopeId: string; exception: SentryErrorEvent["exception"] } + >(); + let currentDiagnostics: Diagnostic[] = []; + const stableCopies = new WeakMap(); + function active(epoch: number) { + return !disposed && epoch === generation; + } + function diagnostic( + code: DiagnosticCode, + message: string, + details: Partial> = {}, + ) { + const item = makeDiagnostic(code, message, details); + currentDiagnostics.push(item); + reportDiagnostic(options.onDiagnostic, item); + return item; + } + function invalid( + code: DiagnosticCode, + message: string, + diagnostics?: readonly Diagnostic[], + ): IngestResult { + const items = diagnostics ?? [makeDiagnostic(code, message)]; + for (const item of items) reportDiagnostic(options.onDiagnostic, item); + return receipt({ status: "invalid", diagnostics: items }, { revision }); + } + function ignored( + reason: IgnoreReason, + envelopeId?: string, + retained = false, + ): IngestResult { + return receipt( + { + status: "ignored", + reason, + ...(envelopeId !== undefined ? { envelopeId } : {}), + }, + { revision, retained }, + ); + } + function makeEngine() { + const cache = enrichmentCache; + const epoch = generation; + const lifetimeSignal = lifetime.signal; + let currentEnvelopeId = ""; + let currentSignal: AbortSignal | undefined; + const created = createEngine({ + enrichStacktrace: async (event) => { + const envelopeId = currentEnvelopeId; + const inputSignal = currentSignal; + const cached = cache.get(event.event_id); + if (cached?.envelopeId === envelopeId) { + event.exception = structuredClone(cached.exception); + return; + } + if (options.enrichStacktrace) { + // A user hook receives an isolated copy. After timeout/abort, late writes cannot escape. + const enriched = structuredClone(event); + const controller = new AbortController(); + let timeout: ReturnType | undefined; + let completed = false; + let abortListener!: () => void; + let lifetimeListener!: () => void; + try { + await new Promise((resolve) => { + const finish = () => { + if (!completed) { + completed = true; + resolve(); + } + }; + const cancel = () => { + controller.abort(); + finish(); + }; + abortListener = cancel; + lifetimeListener = cancel; + inputSignal?.addEventListener("abort", abortListener, { + once: true, + }); + lifetimeSignal.addEventListener("abort", lifetimeListener, { + once: true, + }); + if (inputSignal?.aborted || lifetimeSignal.aborted) { + cancel(); + return; + } + const deadline = Date.now() + enrichmentTimeoutMs; + const checkDeadline = () => { + if (completed) return; + const remaining = deadline - Date.now(); + if (remaining > 0) { + timeout = setTimeout( + checkDeadline, + Math.min(2147483647, remaining), + ); + return; + } + if (active(epoch)) + diagnostic( + "enrichment_timeout", + "Stacktrace enrichment exceeded its deadline; telemetry was retained.", + { severity: "warning" }, + ); + cancel(); + }; + checkDeadline(); + void Promise.resolve() + .then(() => + options.enrichStacktrace!(enriched, { + signal: controller.signal, + }), + ) + .then( + () => { + if ( + !completed && + active(epoch) && + created.runtime.generation === 0 && + !controller.signal.aborted + ) + event.exception = structuredClone(enriched.exception); + finish(); + }, + () => { + if (!completed && active(epoch)) + diagnostic( + "enrichment_failed", + "Stacktrace enrichment failed; telemetry was retained.", + { + severity: "warning", + }, + ); + finish(); + }, + ); + }); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + inputSignal?.removeEventListener("abort", abortListener); + lifetimeSignal.removeEventListener("abort", lifetimeListener); + } + } + if ( + active(epoch) && + created.runtime.generation === 0 && + !inputSignal?.aborted + ) + cache.set(event.event_id, { + envelopeId, + exception: structuredClone(event.exception), + }); + }, + }); + return { + ...created, + ingestEnvelope( + envelope: Envelope, + receivedAtMs: number, + signal?: AbortSignal, + ) { + currentEnvelopeId = stringifyValue(envelope[0].__spotlight_envelope_id); + currentSignal = signal; + created.runtime.receivedAtMs = receivedAtMs; + return created.store.getState().ingestEnvelope(envelope); + }, + }; + } + let engine = makeEngine(); + function stable(value: T): T { + if (!stableCopies.has(value)) stableCopies.set(value, readonlyCopy(value)); + return stableCopies.get(value) as T; + } + function stateSnapshot(): TelemetrySnapshot { + const state = engine.store.getState(); + return Object.freeze({ + revision, + envelopes: Object.freeze(state.getEnvelopes().map(stable)), + events: Object.freeze(state.getEvents().map(stable)), + traces: readonlyCopy(state.getTraces()), + logs: Object.freeze(state.getLogs().map(stable)), + sdks: readonlyCopy(state.getSdks()), + profiles: readonlyCopy(state.profilesByTraceId), + }); + } + let snapshot = stateSnapshot(); + function publish( + type: ChangeMetadata["type"], + envelopeIds: readonly string[] = [], + ) { + revision++; + snapshot = stateSnapshot(); + const committed = snapshot; + const change = Object.freeze({ + type, + revision, + envelopeIds: Object.freeze([...envelopeIds]), + }); + for (const listener of listeners) { + try { + listener(committed, change); + } catch { + diagnostic( + "subscriber_failed", + "A telemetry subscription callback failed.", + { severity: "warning" }, + ); + } + if (snapshot !== committed) break; + } + } + function clearTimer() { + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + } + function scheduleExpiry() { + clearTimer(); + if (disposed || maxAgeMs === undefined || !journal.length) return; + const earliest = Math.min(...journal.map((record) => record.receivedAtMs)); + const delay = Math.max( + 1, + Math.min(2147483647, earliest + maxAgeMs - Date.now()), + ); + timer = setTimeout(() => { + timer = undefined; + void client.prune(); + }, delay); + if (typeof timer === "object" && timer && "unref" in timer) timer.unref(); + } + function enqueue(task: (epoch: number) => Promise): Promise { + const epoch = generation; + pending++; + const next = queue + .then(() => task(epoch)) + .finally(() => { + if (epoch === generation) pending--; + }); + queue = next.catch(() => {}); + return next; + } + async function rebuild( + retained: RetainedEnvelope[], + epoch: number, + signal?: AbortSignal, + ) { + const replacement = makeEngine(); + for (const record of retained) { + if (signal?.aborted) return false; + await replacement.ingestEnvelope( + record.envelope, + record.receivedAtMs, + signal, + ); + if (!active(epoch) || signal?.aborted) return false; + } + engine = replacement; + const eventIds = engine.store.getState().eventsById; + for (const key of enrichmentCache.keys()) + if (!eventIds.has(key)) enrichmentCache.delete(key); + return true; + } + async function enforceRetention( + epoch: number, + signal?: AbortSignal, + ): Promise { + const cutoff = + maxAgeMs === undefined ? Number.NEGATIVE_INFINITY : Date.now() - maxAgeMs; + let retained = journal + .filter((record) => record.receivedAtMs > cutoff) + .slice(-maxEnvelopes); + let bytes = retained.reduce((sum, record) => sum + record.bytes, 0); + let trim = 0; + while (bytes > maxBytes && trim < retained.length) + bytes -= retained[trim++].bytes; + retained = retained.slice(trim); + if (retained.length === journal.length) return []; + const keep = new Set(retained.map((record) => record.id)); + const evicted = journal + .filter((record) => !keep.has(record.id)) + .map((record) => record.id); + if (!(await rebuild(retained, epoch, signal))) return []; + if (!active(epoch)) return []; + journal = retained; + return evicted; + } + function spanKeys() { + const keys = new Set(); + for (const trace of engine.store.getState().tracesById.values()) + for (const id of trace.spans.keys()) keys.add(`${trace.trace_id}:${id}`); + return keys; + } + const reader = createTelemetryReader(() => snapshot); + function ingestOne( + input: unknown, + inputOptions: IngestOptions = {}, + withinEpoch?: number, + ): Promise { + const ingestOptions = { + deliveryId: inputOptions.deliveryId, + signal: inputOptions.signal, + }; + if (disposed) return Promise.resolve(ignored("disposed")); + if (ingestOptions.signal?.aborted) + return Promise.resolve(ignored("aborted")); + if (withinEpoch === undefined && pending >= maxPending) + return Promise.resolve(ignored("queue_full")); + if ( + ingestOptions.deliveryId !== undefined && + (typeof ingestOptions.deliveryId !== "string" || + !ingestOptions.deliveryId.trim()) + ) + return Promise.resolve( + invalid( + "invalid_delivery_id", + "deliveryId must be a non-empty string.", + ), + ); + if (!validateEnvelope(input)) + return Promise.resolve( + invalid( + "invalid_envelope", + "Expected a parsed envelope [headers, [[itemHeaders, payload], ...]].", + ), + ); + const supported = supportsEnvelope(input); + const inputDiagnostics: Diagnostic[] = []; + let validSpans = 0; + let invalidSpans = 0; + let otherItems = false; + let candidates = 0; + let skippedItems = 0; + for (const [itemIndex, [header, payload]] of input[1].entries()) { + if (header.type !== "span") { + otherItems = true; + if (SUPPORTED_EVENT_TYPES.has(header.type)) candidates++; + else skippedItems++; + continue; + } + const data = payload as Record; + if ("items" in data && data.version !== 2) { + skippedItems++; + inputDiagnostics.push( + makeDiagnostic( + "unsupported_version", + "Unsupported span container version was retained without processing.", + { + severity: "warning", + itemIndex, + }, + ), + ); + continue; + } + const count = Array.isArray(data.items) ? data.items.length : 1; + const valid = normalizeSpanItem(data, { + name: "unknown", + version: "unknown", + lastSeen: 0, + }).length; + validSpans += valid; + candidates += valid; + invalidSpans += count - valid; + if (count > valid) + inputDiagnostics.push( + makeDiagnostic( + "invalid_span", + "Malformed span items were skipped; valid telemetry was retained.", + { + severity: "warning", + itemIndex, + }, + ), + ); + } + if (invalidSpans > 0 && validSpans === 0 && !otherItems) + return Promise.resolve( + invalid( + "invalid_span", + "Span items are missing valid IDs or timestamps.", + inputDiagnostics.map((item) => + makeDiagnostic( + item.code, + "Span items are missing valid IDs or timestamps.", + { + severity: "error", + itemIndex: item.itemIndex, + }, + ), + ), + ), + ); + skippedItems += invalidSpans; + let id: string; + let envelope: Envelope; + let bytes: number; + try { + id = + ingestOptions.deliveryId ?? + stringifyValue( + input[0].__spotlight_envelope_id ?? + identities.get(input) ?? + generateUuidv4(), + ); + identities.set(input, id); + envelope = normalizeEnvelope(input); + envelope[0].__spotlight_envelope_id = id; + bytes = new TextEncoder().encode(JSON.stringify(envelope)).byteLength; + } catch { + return Promise.resolve( + invalid( + "invalid_envelope", + "Envelope data must be cloneable wire data.", + ), + ); + } + if (bytes > maxEnvelopeBytes) + return Promise.resolve( + invalid( + "payload_too_large", + "Parsed envelope exceeds maxEnvelopeBytes.", + ), + ); + const receivedAtMs = Date.now(); + const process = async (epoch: number): Promise => { + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (ingestOptions.signal?.aborted) return ignored("aborted", id); + currentDiagnostics = []; + for (const item of inputDiagnostics) { + currentDiagnostics.push(item); + reportDiagnostic(options.onDiagnostic, item); + } + const evicted = await enforceRetention(epoch, ingestOptions.signal); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (ingestOptions.signal?.aborted) { + if (evicted.length) publish("evict", evicted); + scheduleExpiry(); + return ignored("aborted", id); + } + const retainedBefore = journal; + const cacheBefore = new Map(enrichmentCache); + async function rollback() { + working.store.getState().resetData(); + if (engine !== working) engine.store.getState().resetData(); + journal = retainedBefore; + enrichmentCache = new Map(cacheBefore); + await rebuild(journal, epoch); + if (active(epoch)) { + if (evicted.length) publish("evict", evicted); + scheduleExpiry(); + } + } + if (journal.some((record) => record.id === id)) { + if (evicted.length) publish("evict", evicted); + scheduleExpiry(); + return ignored("duplicate", id, true); + } + const working = engine; + const beforeEvents = new Set(working.store.getState().eventsById.keys()); + const beforeLogs = new Set(working.store.getState().logsById.keys()); + const beforeSpans = spanKeys(); + let failed = false; + try { + await working.ingestEnvelope( + envelope, + receivedAtMs, + ingestOptions.signal, + ); + } catch { + failed = true; + } + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (failed || ingestOptions.signal?.aborted) { + await rollback(); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + return ingestOptions.signal?.aborted + ? ignored("aborted", id) + : invalid( + "invalid_envelope", + "Envelope telemetry could not be processed.", + ); + } + const state = working.store.getState(); + const addedEvents = Array.from(state.eventsById.keys()).filter( + (key) => !beforeEvents.has(key), + ).length; + const counts: IngestCounts = { + events: addedEvents, + logs: Array.from(state.logsById.keys()).filter( + (key) => !beforeLogs.has(key), + ).length, + spans: Array.from(spanKeys()).filter((key) => !beforeSpans.has(key)) + .length, + skippedItems, + duplicateEvents: Math.max(0, candidates - addedEvents), + }; + journal = [...journal, { id, envelope, receivedAtMs, bytes }]; + const afterEvicted = await enforceRetention(epoch, ingestOptions.signal); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (ingestOptions.signal?.aborted) { + await rollback(); + return ignored( + active(epoch) ? "aborted" : disposed ? "disposed" : "reset", + id, + ); + } + publish("ingest", [id, ...evicted, ...afterEvicted]); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + scheduleExpiry(); + return receipt( + supported + ? { + status: "processed", + envelopeId: id, + diagnostics: currentDiagnostics, + } + : { + status: "ignored", + reason: "unsupported_items", + envelopeId: id, + diagnostics: currentDiagnostics, + }, + { + revision, + retained: journal.some((record) => record.id === id), + counts, + }, + ); + }; + return withinEpoch === undefined ? enqueue(process) : process(withinEpoch); + } + function queueRetentionPrune(): Promise { + const epoch = generation; + const operation = queue.then(async () => { + if (!active(epoch)) return; + currentDiagnostics = []; + const evicted = await enforceRetention(epoch); + if (active(epoch) && evicted.length) publish("evict", evicted); + if (active(epoch)) scheduleExpiry(); + }); + queue = operation.catch(() => {}); + return operation; + } + const client: TelemetryStore = { + ...reader, + ingestEnvelope: (input, inputOptions) => ingestOne(input, inputOptions), + ingestEnvelopes(inputs, batchOptions = {}) { + const signal = batchOptions.signal; + if (disposed || pending >= maxPending) { + return Promise.resolve( + Object.freeze( + Array.from(inputs, () => + ignored(disposed ? "disposed" : "queue_full"), + ), + ), + ); + } + return enqueue(async (epoch) => { + const results: IngestResult[] = []; + for (const input of inputs) { + results.push( + active(epoch) + ? await ingestOne(input, { signal }, epoch) + : ignored(disposed ? "disposed" : "reset"), + ); + } + return Object.freeze(results); + }); + }, + drain: () => queue.then(() => {}), + getStats: () => + Object.freeze({ + pending, + retainedEnvelopes: journal.length, + retainedBytes: journal.reduce((sum, item) => sum + item.bytes, 0), + revision, + disposed, + }), + subscribe(listener) { + if (disposed) return () => {}; + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + reset() { + if (disposed) return; + generation++; + lifetime.abort(); + lifetime = new AbortController(); + clearTimer(); + engine.store.getState().resetData(); + journal = []; + enrichmentCache = new Map(); + engine = makeEngine(); + queue = Promise.resolve(); + pruning = undefined; + pending = 0; + publish("reset"); + }, + setRetention(update) { + if (disposed) return Promise.resolve(); + const next = validateRetention(update); + if (next.maxEnvelopes !== undefined) maxEnvelopes = next.maxEnvelopes; + if (next.maxBytes !== undefined) maxBytes = next.maxBytes; + if (next.maxAgeMs !== undefined) maxAgeMs = next.maxAgeMs; + // A distinct barrier must cover work submitted after an earlier coalesced prune. + return queueRetentionPrune(); + }, + prune() { + if (disposed) return Promise.resolve(); + if (pruning) return pruning; + // Control work has one coalesced continuation outside ingestion admission. + // Link it into the same barrier so even invalid/empty work cannot skip expiry. + const operation = queueRetentionPrune(); + pruning = operation; + void operation.then( + () => { + if (pruning === operation) pruning = undefined; + }, + () => { + if (pruning === operation) pruning = undefined; + }, + ); + return operation; + }, + dispose() { + if (disposed) return; + generation++; + disposed = true; + lifetime.abort(); + clearTimer(); + engine.store.getState().resetData(); + journal = []; + enrichmentCache.clear(); + engine = makeEngine(); + pending = 0; + publish("dispose"); + listeners.clear(); + pruning = undefined; + queue = Promise.resolve(); + }, + }; + return client; +} +export type { + TelemetryStore, + TelemetryStoreOptions, + TelemetrySnapshot, +} from "./public.ts"; +export type { IngestResult, Diagnostic } from "./contracts.ts"; diff --git a/packages/telemetry/src/telemetry/inspection-models.spec.ts b/packages/telemetry/src/telemetry/inspection-models.spec.ts new file mode 100644 index 000000000..dc88e71d3 --- /dev/null +++ b/packages/telemetry/src/telemetry/inspection-models.spec.ts @@ -0,0 +1,108 @@ +import { expect, test } from "vitest"; +import { + buildProfileModel, + groupWebVitalPages, + inspectAI, + readSpanData, +} from "../sdk/core.ts"; + +test("profile model calculates intervals independently per thread and preserves leaf/self counts", () => { + const payload = { + profile: { + frames: [ + { function: "main", filename: "app.ts", in_app: true }, + { function: "query" }, + ], + stacks: [[1, 0], [0]], + samples: [ + { thread_id: "main", stack_id: 0, elapsed_since_start_ns: "0" }, + { thread_id: "worker", stack_id: 1, elapsed_since_start_ns: "0" }, + { thread_id: "main", stack_id: 1, elapsed_since_start_ns: "10000000" }, + { + thread_id: "worker", + stack_id: 1, + elapsed_since_start_ns: "90000000", + }, + ], + thread_metadata: { main: { name: "UI thread" } }, + }, + }; + const before = structuredClone(payload); + const model = buildProfileModel(payload, "main"); + expect(model.durationMs).toBe(10); + expect(model.sampleCount).toBe(2); + expect(model.root.children[0]).toMatchObject({ + name: "main", + samples: 2, + selfSamples: 1, + children: [{ name: "query", selfMs: 10 }], + }); + expect(buildProfileModel(payload, "worker").durationMs).toBe(90); + expect(buildProfileModel(payload, "missing").threadId).toBe("main"); + expect(payload).toEqual(before); +}); + +test("AI inspection reads typed span-v2 aliases, zero token counts, JSON messages and tool execution", () => { + const payload = { + attributes: { + "sentry.op": { type: "string", value: "gen_ai.execute_tool" }, + "gen_ai.request.model": { type: "string", value: "test-model" }, + "gen_ai.usage.input_tokens": { type: "integer", value: 0 }, + "gen_ai.request.messages": { + type: "string", + value: '[{"role":"user","content":"Hello"}]', + }, + "gen_ai.tool.name": { type: "string", value: "weather" }, + "gen_ai.tool.input": { type: "string", value: '{"city":"Pune"}' }, + "gen_ai.tool.output": { type: "string", value: '{"temperature":28}' }, + }, + }; + expect(inspectAI(payload)).toMatchObject({ + fields: { model: "test-model", inputTokens: 0 }, + messages: [{ role: "user", content: "Hello" }], + tools: [ + { + name: "weather", + arguments: { city: "Pune" }, + result: { temperature: 28 }, + }, + ], + }); + expect( + inspectAI({ op: "db.query", data: { sql: "select 1" } }), + ).toBeUndefined(); + expect( + readSpanData({ data: { custom: "retained" }, ...payload }).custom, + ).toBe("retained"); +}); + +test("Web Vitals preserve absent metrics, explicit seconds, zero values and separate visits", () => { + const pages = groupWebVitalPages([ + { + id: "one", + transaction: "/orders", + measurements: { lcp: { value: 3.2, unit: "second" }, cls: { value: 0 } }, + }, + { + id: "two", + transaction: "/orders", + measurements: { inp: { value: 700 }, fcp: { value: -1 } }, + }, + { + id: "invalid", + transaction: "/unknown", + measurements: { lcp: { value: "200" } }, + }, + ]); + expect(pages).toHaveLength(1); + expect(pages[0]?.visits).toEqual([ + { + eventId: "one", + metrics: { + lcp: { value: 3200, rating: "Needs improvement" }, + cls: { value: 0, rating: "Good" }, + }, + }, + { eventId: "two", metrics: { inp: { value: 700, rating: "Poor" } } }, + ]); +}); diff --git a/packages/telemetry/src/telemetry/internal/engine.ts b/packages/telemetry/src/telemetry/internal/engine.ts new file mode 100644 index 000000000..cb0600815 --- /dev/null +++ b/packages/telemetry/src/telemetry/internal/engine.ts @@ -0,0 +1,30 @@ +import { createStore } from "zustand/vanilla"; +import { createEnvelopesSlice } from "../store/slices/envelopesSlice.ts"; +import { createEventsSlice } from "../store/slices/eventsSlice.ts"; +import { createLogsSlice } from "../store/slices/logsSlice.ts"; +import { createProfilesSlice } from "../store/slices/profilesSlice.ts"; +import { createSDKsSlice } from "../store/slices/sdksSlice.ts"; +import { createSharedSlice } from "../store/slices/sharedSlice.ts"; +import { createSubscriptionsSlice } from "../store/slices/subscriptionsSlice.ts"; +import { createTracesSlice } from "../store/slices/tracesSlice.ts"; +import type { SentryStore } from "../store/types.ts"; + +import type { EngineOptions, ProcessingRuntime } from "./types.ts"; + +export function createEngine(options: EngineOptions = {}) { + const runtime: ProcessingRuntime = { + generation: 0, + receivedAtMs: Date.now(), + }; + const store = createStore()((...args) => ({ + ...createEventsSlice(options, runtime)(...args), + ...createTracesSlice(...args), + ...createProfilesSlice(...args), + ...createSubscriptionsSlice(...args), + ...createEnvelopesSlice(runtime)(...args), + ...createSDKsSlice(...args), + ...createLogsSlice(...args), + ...createSharedSlice(runtime)(...args), + })); + return { store, runtime }; +} diff --git a/packages/telemetry/src/telemetry/internal/envelopes.ts b/packages/telemetry/src/telemetry/internal/envelopes.ts new file mode 100644 index 000000000..6d5fed235 --- /dev/null +++ b/packages/telemetry/src/telemetry/internal/envelopes.ts @@ -0,0 +1,53 @@ +import type { Envelope } from "@sentry/core"; +import { SUPPORTED_EVENT_TYPES } from "../constants/sentry.ts"; +import { opaquePayload } from "../envelope/payload.ts"; + +export function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} +export function validateEnvelope(input: unknown): input is Envelope { + if ( + !Array.isArray(input) || + input.length !== 2 || + !isRecord(input[0]) || + !Array.isArray(input[1]) + ) + return false; + return input[1].every( + (item) => + Array.isArray(item) && + item.length === 2 && + isRecord(item[0]) && + typeof item[0].type === "string" && + ((isRecord(item[1]) && !(item[1] instanceof Uint8Array)) || + (!SUPPORTED_EVENT_TYPES.has(item[0].type) && + item[0].type !== "span" && + (typeof item[1] === "string" || item[1] instanceof Uint8Array))), + ); +} + +/** Match serialized opaque items with the shared portable byte policy. */ +export function normalizeEnvelope(input: Envelope): Envelope { + const envelope = structuredClone(input); + for (const item of envelope[1]) { + const [header, payload] = item; + if (typeof payload !== "string" && !(payload instanceof Uint8Array)) + continue; + if (!header.content_type && (header.type as string) === "statsd") + header.content_type = "text/plain"; + item[1] = opaquePayload( + payload, + header.content_type as string, + ) as (typeof item)[1]; + } + return envelope; +} +export function supportsEnvelope(input: Envelope): boolean { + return input[1].some(([header, payload]) => { + if (header.type !== "span") return SUPPORTED_EVENT_TYPES.has(header.type); + const span = payload as Record; + return ( + !("items" in span) || (span.version === 2 && Array.isArray(span.items)) + ); + }); +} diff --git a/packages/telemetry/src/telemetry/internal/readonly.spec.ts b/packages/telemetry/src/telemetry/internal/readonly.spec.ts new file mode 100644 index 000000000..df923ff62 --- /dev/null +++ b/packages/telemetry/src/telemetry/internal/readonly.spec.ts @@ -0,0 +1,30 @@ +import { expect, test } from "vitest"; +import { readonlyCopy } from "./readonly.ts"; + +test("copies deep graphs without recursion while preserving aliases and cycles", () => { + type Link = { next?: Link; root?: Link }; + const root: Link = {}; + let tail = root; + for (let i = 0; i < 10000; i++) { + tail.next = {}; + tail = tail.next; + } + tail.root = root; + const input = { root, aliases: new Map([[root, new Set([tail])]]) }; + const copied = readonlyCopy(input); + let current = copied.root; + for (let i = 0; i < 10000; i++) { + expect(Object.isFrozen(current)).toBe(true); + current = current.next!; + } + expect(current.root === copied.root).toBe(true); + expect(copied.aliases.get(copied.root)?.has(current)).toBe(true); + expect(() => (copied.aliases as Map>).clear()).toThrow( + TypeError, + ); + expect(() => + (copied.aliases.get(copied.root) as Set).add(root), + ).toThrow(TypeError); + expect(root === copied.root).toBe(false); + expect(Object.isFrozen(root)).toBe(false); +}); diff --git a/packages/telemetry/src/telemetry/internal/readonly.ts b/packages/telemetry/src/telemetry/internal/readonly.ts new file mode 100644 index 000000000..7ce4ddaf0 --- /dev/null +++ b/packages/telemetry/src/telemetry/internal/readonly.ts @@ -0,0 +1,85 @@ +import type { DeepReadonly } from "../public.ts"; + +/** Copies the graph before freezing it; Map/Set mutation methods never expose the backing collection. */ +export function readonlyCopy(input: T): DeepReadonly { + const copies = new WeakMap(); + const pending: (() => void)[] = []; + function copy(value: unknown): unknown { + if (!value || typeof value !== "object") return value; + if (copies.has(value)) return copies.get(value); + if (value instanceof Map || value instanceof Set) { + const target = value instanceof Map ? new Map() : new Set(); + const proxy = new Proxy(target, { + get(collection, key) { + if (["set", "add", "delete", "clear"].includes(String(key))) + return () => { + throw new TypeError("Telemetry snapshots are read-only"); + }; + if (key === "valueOf") return () => proxy; + if (key === "forEach") + return ( + callback: ( + value: unknown, + key: unknown, + collection: unknown, + ) => void, + thisArg?: unknown, + ) => { + collection.forEach((item, itemKey) => + callback.call(thisArg, item, itemKey, proxy), + ); + }; + const member = Reflect.get(collection, key, collection); + return typeof member === "function" && + (["get", "has", "entries", "keys", "values"].includes( + String(key), + ) || + key === Symbol.iterator) + ? member.bind(collection) + : member; + }, + setPrototypeOf() { + throw new TypeError("Telemetry snapshots are read-only"); + }, + set() { + throw new TypeError("Telemetry snapshots are read-only"); + }, + defineProperty() { + throw new TypeError("Telemetry snapshots are read-only"); + }, + deleteProperty() { + throw new TypeError("Telemetry snapshots are read-only"); + }, + }); + copies.set(value, proxy); + pending.push(() => { + if (value instanceof Map && target instanceof Map) + for (const [key, item] of value) target.set(copy(key), copy(item)); + else if (value instanceof Set && target instanceof Set) + for (const item of value) target.add(copy(item)); + }); + return proxy; + } + const result: Record | unknown[] = Array.isArray( + value, + ) + ? [] + : {}; + copies.set(value, result); + pending.push(() => { + for (const key of Object.keys(value)) + Object.defineProperty(result, key, { + value: copy((value as Record)[key]), + enumerable: true, + configurable: true, + writable: true, + }); + Object.freeze(result); + }); + return result; + } + const result = copy(input); + // Allocate each node once, then fill it from a worklist instead of the call stack. + for (let next = pending.pop(); next; next = pending.pop()) next(); + return result as DeepReadonly; +} diff --git a/packages/telemetry/src/telemetry/internal/results.ts b/packages/telemetry/src/telemetry/internal/results.ts new file mode 100644 index 000000000..f63362add --- /dev/null +++ b/packages/telemetry/src/telemetry/internal/results.ts @@ -0,0 +1,50 @@ +import type { + Diagnostic, + DiagnosticCode, + IngestCounts, + IngestOutcome, + IngestResult, +} from "../contracts.ts"; + +export const EMPTY_COUNTS: IngestCounts = Object.freeze({ + events: 0, + spans: 0, + logs: 0, + skippedItems: 0, + duplicateEvents: 0, +}); +export function diagnostic( + code: DiagnosticCode, + message: string, + details: Partial> = {}, +): Diagnostic { + return Object.freeze({ code, message, severity: "error", ...details }); +} +export function reportDiagnostic( + observer: ((item: Diagnostic) => void) | undefined, + item: Diagnostic, +) { + try { + observer?.(item); + } catch { + /* Observers never change capture outcomes. */ + } +} +export function receipt( + outcome: IngestOutcome, + metadata: { + revision: number; + retained?: boolean; + counts?: Partial; + }, +): IngestResult { + return Object.freeze({ + ...outcome, + diagnostics: Object.freeze( + (outcome.diagnostics ?? []).map((item) => Object.freeze({ ...item })), + ), + revision: metadata.revision, + retained: metadata.retained ?? false, + counts: Object.freeze({ ...EMPTY_COUNTS, ...metadata.counts }), + }) as IngestResult; +} diff --git a/packages/telemetry/src/telemetry/internal/retention.ts b/packages/telemetry/src/telemetry/internal/retention.ts new file mode 100644 index 000000000..7130c75cd --- /dev/null +++ b/packages/telemetry/src/telemetry/internal/retention.ts @@ -0,0 +1,16 @@ +import type { RetentionOptions } from "../public.ts"; + +/** Copy and validate the complete partial update before any live limits change. */ +export function validateRetention(options: RetentionOptions): RetentionOptions { + const { maxEnvelopes, maxBytes, maxAgeMs } = options; + for (const [name, value] of [ + ["maxEnvelopes", maxEnvelopes], + ["maxBytes", maxBytes], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) + throw new RangeError(`retention.${name} must be a positive safe integer`); + } + if (maxAgeMs !== undefined && (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0)) + throw new RangeError("retention.maxAgeMs must be a positive finite number"); + return { maxEnvelopes, maxBytes, maxAgeMs }; +} diff --git a/packages/telemetry/src/telemetry/internal/types.ts b/packages/telemetry/src/telemetry/internal/types.ts new file mode 100644 index 000000000..88cab729c --- /dev/null +++ b/packages/telemetry/src/telemetry/internal/types.ts @@ -0,0 +1,5 @@ +import type { SentryErrorEvent } from "../types.ts"; +export type EngineOptions = { + enrichStacktrace?: (event: SentryErrorEvent) => Promise; +}; +export type ProcessingRuntime = { generation: number; receivedAtMs: number }; diff --git a/packages/telemetry/src/telemetry/internal/wireValues.ts b/packages/telemetry/src/telemetry/internal/wireValues.ts new file mode 100644 index 000000000..e8a8b278c --- /dev/null +++ b/packages/telemetry/src/telemetry/internal/wireValues.ts @@ -0,0 +1,22 @@ +/** Read forward-compatible wire fields without coercing invalid values. */ +export function record(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} +export function text(value: unknown): string | undefined { + return typeof value === "string" && value ? value : undefined; +} +export function finite(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} +export function parsed(value: unknown): unknown { + if (typeof value !== "string") return value; + try { + return JSON.parse(value); + } catch { + return value; + } +} diff --git a/packages/telemetry/src/telemetry/lifecycle.spec.ts b/packages/telemetry/src/telemetry/lifecycle.spec.ts new file mode 100644 index 000000000..bc3238660 --- /dev/null +++ b/packages/telemetry/src/telemetry/lifecycle.spec.ts @@ -0,0 +1,463 @@ +// @vitest-environment node +import { afterEach, expect, test, vi } from "vitest"; +import { createTelemetryPipeline } from "../sdk/node.ts"; +import { createTelemetryStore } from "./index.ts"; + +const event = (id = "event") => [ + {}, + [[{ type: "event" }, { event_id: id, timestamp: 1 }]], +]; +const error = (id = "error") => [ + {}, + [ + [ + { type: "event" }, + { + event_id: id, + timestamp: 1, + exception: { values: [{ type: "Error", value: "original" }] }, + }, + ], + ], +]; +afterEach(() => vi.useRealTimers()); + +test.each(["core", "node"] as const)( + "%s pruning after full-queue no-op work still enforces expiry", + async (runtime) => { + vi.useFakeTimers(); + const client = + runtime === "core" + ? createTelemetryStore({ maxPending: 1, retention: { maxAgeMs: 1000 } }) + : createTelemetryPipeline({ + maxPending: 1, + retention: { maxAgeMs: 1000 }, + }); + await client.ingestEnvelope(event()); + vi.setSystemTime(Date.now() + 10000); + const work = + "ingestPayload" in client + ? client.ingestPayload({ body: "bad" }) + : client.ingestEnvelopes([]); + await Promise.all([work, client.prune()]); + expect(client.getEnvelopes()).toEqual([]); + expect(client.getStats().pending).toBe(0); + client.dispose(); + }, +); + +test("pruning is coalesced and cannot bypass the configured pending limit", async () => { + const ingester = createTelemetryStore({ maxPending: 1 }); + const operations = Array.from({ length: 100 }, () => ingester.prune()); + expect(ingester.getStats().pending).toBeLessThanOrEqual(1); + expect(operations[0]).toBe(operations[99]); + await Promise.all(operations); + expect(ingester.getStats().pending).toBe(0); + ingester.dispose(); +}); + +test("large enrichment deadlines do not overflow Node timers", async () => { + vi.useFakeTimers(); + const ingester = createTelemetryStore({ + enrichmentTimeoutMs: 2147483648, + enrichStacktrace: async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }, + }); + const pending = ingester.ingestEnvelope(error()); + await vi.advanceTimersByTimeAsync(10); + expect(await pending).toMatchObject({ status: "processed", diagnostics: [] }); + ingester.dispose(); +}); + +test("receipts distinguish retained unsupported data and no-op duplicates", async () => { + const ingester = createTelemetryStore(); + const first = await ingester.ingestEnvelope(event(), { + deliveryId: "delivery", + }); + expect(first).toMatchObject({ + status: "processed", + envelopeId: "delivery", + retained: true, + revision: 1, + counts: { events: 1, duplicateEvents: 0 }, + }); + expect( + await ingester.ingestEnvelope(event(), { deliveryId: "delivery" }), + ).toMatchObject({ + status: "ignored", + reason: "duplicate", + retained: true, + revision: 1, + }); + expect( + await ingester.ingestEnvelope([{}, [[{ type: "future" }, {}]]]), + ).toMatchObject({ + status: "ignored", + reason: "unsupported_items", + retained: true, + revision: 2, + counts: { skippedItems: 1 }, + }); + expect(Object.isFrozen(first.counts)).toBe(true); + ingester.dispose(); +}); + +test("batch ingestion preserves order and a drain barrier covers earlier work", async () => { + const ingester = createTelemetryStore(); + const batch = ingester.ingestEnvelopes([event("one"), null, event("two")]); + await ingester.drain(); + expect((await batch).map((result) => result.status)).toEqual([ + "processed", + "invalid", + "processed", + ]); + expect(ingester.getEvents().map((item) => item.event_id)).toEqual([ + "one", + "two", + ]); + expect(ingester.getStats().pending).toBe(0); + ingester.dispose(); +}); + +test("batches are one queued operation and drain does not wait for later submissions", async () => { + let releaseFirst!: () => void; + let releaseLater!: () => void; + let enteredFirst!: () => void; + const ready = new Promise((resolve) => { + enteredFirst = resolve; + }); + const ingester = createTelemetryStore({ + enrichStacktrace: async (item) => { + if (item.event_id === "first") { + enteredFirst(); + await new Promise((resolve) => { + releaseFirst = resolve; + }); + } + if (item.event_id === "later") + await new Promise((resolve) => { + releaseLater = resolve; + }); + }, + }); + const batch = ingester.ingestEnvelopes([error("first"), event("second")]); + const barrier = ingester.drain(); + const later = ingester.ingestEnvelope(error("later")); + await ready; + releaseFirst(); + await barrier; + expect((await batch).map((result) => result.status)).toEqual([ + "processed", + "processed", + ]); + expect(ingester.getEvent("second")).toBeDefined(); + // The later hook may start, but it cannot keep the earlier barrier pending. + await vi.waitFor(() => expect(releaseLater).toBeTypeOf("function")); + releaseLater(); + await later; + ingester.dispose(); +}); + +test("a timed out hook cannot mutate committed telemetry or block later work", async () => { + vi.useFakeTimers(); + let finish!: () => void; + const ingester = createTelemetryStore({ + enrichmentTimeoutMs: 20, + enrichStacktrace: async (item) => { + await new Promise((resolve) => { + finish = resolve; + }); + item.exception.values![0].value = "late"; + }, + }); + const captured = ingester.ingestEnvelope(error()); + const later = ingester.ingestEnvelope(event("later")); + await vi.advanceTimersByTimeAsync(20); + expect(await captured).toMatchObject({ + status: "processed", + diagnostics: [{ code: "enrichment_timeout", severity: "warning" }], + }); + expect((await later).status).toBe("processed"); + finish(); + await Promise.resolve(); + expect(ingester.getErrors()[0].exception.values![0].value).toBe("original"); + ingester.dispose(); +}); + +test("queue capacity and abort have structured outcomes and release capacity", async () => { + let entered!: () => void; + const ready = new Promise((resolve) => { + entered = resolve; + }); + const ingester = createTelemetryStore({ + maxPending: 1, + enrichStacktrace: async (_item, { signal }) => { + entered(); + await new Promise((resolve) => + signal.addEventListener("abort", () => resolve(), { once: true }), + ); + }, + }); + const controller = new AbortController(); + const active = ingester.ingestEnvelope(error(), { + signal: controller.signal, + }); + await ready; + expect(await ingester.ingestEnvelope(event())).toMatchObject({ + status: "ignored", + reason: "queue_full", + retained: false, + }); + controller.abort(); + expect(await active).toMatchObject({ status: "ignored", reason: "aborted" }); + expect(ingester.getEvents()).toEqual([]); + expect((await ingester.ingestEnvelope(event())).status).toBe("processed"); + ingester.dispose(); +}); + +test("abort during post-ingestion retention replay restores the prior committed history", async () => { + let replaying!: () => void; + const replayReady = new Promise((resolve) => { + replaying = resolve; + }); + let calls = 0; + let replaySignal: AbortSignal | undefined; + const ingester = createTelemetryStore({ + retention: { maxEnvelopes: 1 }, + enrichStacktrace: async (_event, { signal }) => { + if (++calls === 1) return; + replaySignal = signal; + replaying(); + await new Promise((resolve) => + signal.addEventListener("abort", () => resolve(), { once: true }), + ); + }, + }); + await ingester.ingestEnvelope(error("same"), { deliveryId: "one" }); + const snapshot = ingester.getSnapshot(); + const controller = new AbortController(); + const aborted = ingester.ingestEnvelope(error("same"), { + deliveryId: "two", + signal: controller.signal, + }); + await replayReady; + controller.abort(); + expect(await aborted).toMatchObject({ + status: "ignored", + reason: "aborted", + retained: false, + }); + expect(replaySignal?.aborted).toBe(true); + expect(ingester.getSnapshot()).toBe(snapshot); + expect(ingester.getEnvelope("one")).toBeDefined(); + expect(ingester.getEnvelope("two")).toBeUndefined(); + expect(calls).toBe(2); + ingester.dispose(); +}); + +test("byte retention evicts dependent data and delivery keys have the same lifetime", async () => { + const ingester = createTelemetryStore({ retention: { maxBytes: 100 } }); + await ingester.ingestEnvelope(event("one"), { deliveryId: "one" }); + await ingester.ingestEnvelope(event("two"), { deliveryId: "two" }); + expect(ingester.getStats().retainedBytes).toBeLessThanOrEqual(100); + expect(ingester.getEvent("one")).toBeUndefined(); + expect( + (await ingester.ingestEnvelope(event("one"), { deliveryId: "one" })).status, + ).toBe("processed"); + ingester.dispose(); +}); + +test("changes identify ingestion, eviction and reset while callbacks see committed reads", async () => { + const ingester = createTelemetryStore({ retention: { maxEnvelopes: 1 } }); + const changes: string[] = []; + ingester.subscribe((snapshot, change) => { + expect(snapshot).toBe(ingester.getSnapshot()); + changes.push(change.type); + }); + await ingester.ingestEnvelope(event("one")); + await ingester.ingestEnvelope(event("two")); + ingester.reset(); + expect(changes).toEqual(["ingest", "ingest", "reset"]); + ingester.dispose(); +}); + +test.each(["core", "node"] as const)( + "%s live retention preserves subscriptions and applies partial limits", + async (runtime) => { + const create = + runtime === "core" ? createTelemetryStore : createTelemetryPipeline; + const client = create({ retention: { maxEnvelopes: 1 } }); + const changes: string[] = []; + const unsubscribe = client.subscribe((_snapshot, change) => + changes.push(change.type), + ); + await client.ingestEnvelope(event("one")); + await client.setRetention({ maxEnvelopes: 3 }); + await client.ingestEnvelope(event("two")); + await client.ingestEnvelope(event("three")); + expect( + client + .getEvents() + .map((item) => item.event_id) + .sort(), + ).toEqual(["one", "three", "two"]); + await client.setRetention({ maxEnvelopes: 2 }); + expect( + client + .getEvents() + .map((item) => item.event_id) + .sort(), + ).toEqual(["three", "two"]); + expect(changes).toEqual(["ingest", "ingest", "ingest", "evict"]); + await client.setRetention({ maxBytes: 1 }); + expect(client.getEnvelopes()).toEqual([]); + await client.setRetention({ maxBytes: 10000 }); + await client.ingestEnvelope(event("four")); + await client.ingestEnvelope(event("five")); + await client.ingestEnvelope(event("six")); + expect(client.getEvents().map((item) => item.event_id)).toEqual([ + "five", + "six", + ]); + unsubscribe(); + client.dispose(); + }, +); + +test.each(["core", "node"] as const)( + "%s live age retention reschedules expiry and validates updates atomically", + async (runtime) => { + vi.useFakeTimers(); + const create = + runtime === "core" ? createTelemetryStore : createTelemetryPipeline; + const client = create({ retention: { maxEnvelopes: 3, maxAgeMs: 1000 } }); + await client.ingestEnvelope(event("one")); + await client.ingestEnvelope(event("two")); + for (const update of [ + { maxEnvelopes: 1, maxBytes: 0 }, + { maxEnvelopes: 1, maxAgeMs: Number.NaN }, + { maxEnvelopes: 0 }, + ]) { + expect(() => client.setRetention(update)).toThrow(RangeError); + } + await client.prune(); + expect(client.getEnvelopes()).toHaveLength(2); + vi.setSystemTime(Date.now() + 100); + await client.setRetention({ maxAgeMs: 200 }); + expect(client.getEnvelopes()).toHaveLength(2); + await vi.advanceTimersByTimeAsync(101); + expect(client.getEnvelopes()).toEqual([]); + await client.ingestEnvelope(event("three")); + await client.setRetention({ maxEnvelopes: 2 }); + await vi.advanceTimersByTimeAsync(201); + expect(client.getEnvelopes()).toEqual([]); + client.dispose(); + await expect( + client.setRetention({ maxEnvelopes: 0 }), + ).resolves.toBeUndefined(); + }, +); + +test("Node retention waits for earlier payloads, preserves decoded history, and participates in drain", async () => { + let releaseFirst!: () => void; + let releaseLater!: () => void; + let enteredFirst!: () => void; + let enteredLater!: () => void; + const ready = new Promise((resolve) => { + enteredFirst = resolve; + }); + const laterReady = new Promise((resolve) => { + enteredLater = resolve; + }); + const client = createTelemetryPipeline({ + retention: { maxEnvelopes: 3 }, + enrichStacktrace: async (item) => { + if (item.event_id === "first") { + enteredFirst(); + await new Promise((resolve) => { + releaseFirst = resolve; + }); + } else if (item.event_id === "later") { + enteredLater(); + await new Promise((resolve) => { + releaseLater = resolve; + }); + } + }, + }); + const wire = (id: string) => + `{}\n{"type":"event"}\n${JSON.stringify(error(id)[1][0][1])}\n`; + const initial = await client.ingestPayload({ body: wire("initial") }); + const changes: number[] = []; + client.subscribe((snapshot) => changes.push(snapshot.envelopes.length)); + const first = client.ingestPayload({ body: wire("first") }); + await ready; + const update = client.setRetention({ maxEnvelopes: 1 }); + const drained = client.drain(); + const later = client.ingestPayload({ body: wire("later") }); + releaseFirst(); + await drained; + await update; + expect((await first).retained).toBe(true); + expect(changes.slice(0, 2)).toEqual([2, 1]); + expect(client.getEvents().map((item) => item.event_id)).toEqual(["first"]); + if (initial.status !== "processed") + throw new Error("Expected processed envelope"); + expect(client.getDecodedPayload(initial.envelopeId)?.body.toString()).toBe( + wire("initial"), + ); + await laterReady; + releaseLater(); + await later; + expect(client.getEvents().map((item) => item.event_id)).toEqual(["later"]); + client.dispose(); +}); + +test("reset invalidates queued Node retention updates while completed limits persist", async () => { + const client = createTelemetryPipeline({ retention: { maxEnvelopes: 3 } }); + const update = client.setRetention({ maxEnvelopes: 1 }); + client.reset(); + await update; + await client.ingestEnvelopes([event("one"), event("two")]); + expect(client.getEnvelopes()).toHaveLength(2); + await client.setRetention({ maxEnvelopes: 1 }); + client.reset(); + await client.ingestEnvelopes([event("three"), event("four")]); + expect(client.getEvents().map((item) => item.event_id)).toEqual(["four"]); + client.dispose(); +}); + +test.each(["core", "node"] as const)( + "%s retention update cannot reuse a prune barrier before later ingestion", + async (runtime) => { + let release!: () => void; + let entered!: () => void; + const ready = new Promise((resolve) => { + entered = resolve; + }); + const create = + runtime === "core" ? createTelemetryStore : createTelemetryPipeline; + const client = create({ + enrichStacktrace: async () => { + entered(); + await new Promise((resolve) => { + release = resolve; + }); + }, + }); + const earlierPrune = client.prune(); + const ingestion = client.ingestEnvelope(error()); + let completed = false; + const update = client.setRetention({ maxEnvelopes: 1 }).then(() => { + completed = true; + }); + await earlierPrune; + await ready; + expect(completed).toBe(false); + release(); + await update; + expect((await ingestion).status).toBe("processed"); + client.dispose(); + }, +); diff --git a/packages/telemetry/src/telemetry/log-traces.spec.ts b/packages/telemetry/src/telemetry/log-traces.spec.ts new file mode 100644 index 000000000..6bc494988 --- /dev/null +++ b/packages/telemetry/src/telemetry/log-traces.spec.ts @@ -0,0 +1,100 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "./index.ts"; + +test("log batches initialize traces from each item's trace ID and normalized timestamp", async () => { + const store = createTelemetryStore(); + const envelope = [ + {}, + [ + [ + { type: "log" }, + { + items: [ + { + id: "one", + trace_id: "logs", + timestamp: 1700000001, + body: "first", + level: "info", + }, + { + id: "two", + trace_id: "other", + timestamp: 1700000002, + body: "other trace", + level: "info", + }, + { + id: "three", + trace_id: "logs", + timestamp: 1700000003, + body: "last", + level: "info", + }, + { + id: "invalid-id", + trace_id: 123, + timestamp: 1700000004, + body: "invalid trace", + level: "info", + }, + { + id: "invalid-time", + trace_id: "bad-time", + timestamp: "invalid", + body: "invalid time", + level: "info", + }, + ], + }, + ], + ], + ]; + const raw = structuredClone(envelope); + await store.ingestEnvelope(envelope); + expect(store.getTrace("logs")).toMatchObject({ + trace_id: "logs", + start_timestamp: 1700000001000, + timestamp: 1700000003000, + spanTree: [], + transactions: [], + }); + expect(store.getTrace("logs")?.spans.size).toBe(0); + expect( + store + .getTraces() + .map((trace) => trace.trace_id) + .sort(), + ).toEqual(["logs", "other"]); + expect(store.getLogs({ traceId: "logs" })).toHaveLength(2); + expect(envelope).toEqual(raw); + + await store.ingestEnvelope([ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "logs", + span_id: "root", + is_segment: true, + name: "GET /later", + start_timestamp: 1700000000, + end_timestamp: 1700000002, + }, + ], + }, + ], + ], + ]); + expect(store.getTrace("logs")).toMatchObject({ + rootTransactionName: "GET /later", + start_timestamp: 1700000000000, + timestamp: 1700000003000, + }); + expect(store.getTrace("logs")?.spans.size).toBe(1); + store.dispose(); +}); diff --git a/packages/telemetry/src/telemetry/logger.ts b/packages/telemetry/src/telemetry/logger.ts new file mode 100644 index 000000000..c9451093d --- /dev/null +++ b/packages/telemetry/src/telemetry/logger.ts @@ -0,0 +1,23 @@ +export const TELEMETRY_PREFIX = "🔎 [Telemetry]"; + +const noop = (..._args: unknown[]) => {}; +let _log = noop; +let _warn = noop; + +export function activateLogger() { + _log = (...args: unknown[]) => console.log(TELEMETRY_PREFIX, ...args); + _warn = (...args: unknown[]) => console.warn(TELEMETRY_PREFIX, ...args); +} + +export function deactivateLogger() { + _log = noop; + _warn = noop; +} + +export function log(...args: unknown[]) { + _log(...args); +} + +export function warn(...args: unknown[]) { + _warn(...args); +} diff --git a/packages/telemetry/src/telemetry/profiles/model.ts b/packages/telemetry/src/telemetry/profiles/model.ts new file mode 100644 index 000000000..a96ce6ef2 --- /dev/null +++ b/packages/telemetry/src/telemetry/profiles/model.ts @@ -0,0 +1,163 @@ +import { record, finite, text } from "../internal/wireValues.ts"; +export type ProfileNode = { + id: string; + name: string; + location: string; + inApp: boolean; + samples: number; + selfSamples: number; + totalMs: number; + selfMs: number; + children: ProfileNode[]; +}; +export type ProfileFunction = Omit; +export type ParsedProfile = { + root: ProfileNode; + functions: ProfileFunction[]; + sampleCount: number; + durationMs?: number; + threads: { id: string; name: string }[]; + threadId?: string; +}; +function node( + id: string, + name: string, + location = "", + inApp = false, +): ProfileNode { + return { + id, + name, + location, + inApp, + samples: 0, + selfSamples: 0, + totalMs: 0, + selfMs: 0, + children: [], + }; +} +function sampleTime(sample: Record): number | undefined { + const ns = sample.elapsed_since_start_ns; + if (typeof ns === "string" && ns.trim() && Number.isFinite(Number(ns))) + return Number(ns) / 1e6; + if (finite(ns) !== undefined) return Number(ns) / 1e6; + const seconds = finite(sample.timestamp); + return seconds === undefined ? undefined : seconds * 1000; +} +function sampleThread(value: unknown): string { + const number = finite(value); + return text(value) ?? (number === undefined ? "main" : String(number)); +} +/** SDK stacks are leaf-first; sample intervals are calculated independently per thread. */ +export function buildProfileModel( + input: unknown, + selectedThread?: string, +): ParsedProfile { + const payload = record(input), + profile = Object.keys(record(payload.profile)).length + ? record(payload.profile) + : payload; + const samples = (Array.isArray(profile.samples) ? profile.samples : []).map( + record, + ); + const frames = Array.isArray(profile.frames) ? profile.frames : []; + const stacks = Array.isArray(profile.stacks) ? profile.stacks : []; + const metadata = record(profile.thread_metadata); + const threads = [ + ...new Set(samples.map((sample) => sampleThread(sample.thread_id))), + ].map((id) => ({ id, name: text(record(metadata[id]).name) ?? id })); + const threadId = threads.some((thread) => thread.id === selectedThread) + ? selectedThread + : threads[0]?.id; + const selected = samples + .filter((sample) => sampleThread(sample.thread_id) === threadId) + .sort((a, b) => (sampleTime(a) ?? Infinity) - (sampleTime(b) ?? Infinity)); + const root = node("root", "All samples"), + functions = new Map(); + const children = new Map>(); + let hasTiming = false; + for (let index = 0; index < selected.length; index++) { + const sample = selected[index]!, + stackId = finite(sample.stack_id), + rawStack: unknown = stackId === undefined ? undefined : stacks[stackId]; + const stack = Array.isArray(rawStack) + ? rawStack + .filter( + (id): id is number => + typeof id === "number" && Number.isInteger(id) && id >= 0, + ) + .toReversed() + : []; + const time = sampleTime(sample), + next = selected[index + 1] ? sampleTime(selected[index + 1]!) : undefined; + const weight = + time !== undefined && next !== undefined ? Math.max(0, next - time) : 0; + if (time !== undefined && next !== undefined) hasTiming = true; + root.samples++; + root.totalMs += weight; + let parent = root; + const seenFunctions = new Set(); + // Bound pathological stack depth to protect the UI; raw data remains inspectable. + for (const frameId of stack.slice(0, 256)) { + if (!frames[frameId] || typeof frames[frameId] !== "object") continue; + const frame = record(frames[frameId]); + const name = + text(frame.function) ?? + text(frame.module) ?? + text(frame.instruction_addr) ?? + "Anonymous"; + const location = [ + text(frame.abs_path) ?? text(frame.filename), + finite(frame.lineno), + ] + .filter((value) => value !== undefined) + .join(":"); + const childMap = children.get(parent) ?? new Map(); + children.set(parent, childMap); + let child = childMap.get(frameId); + if (!child) { + child = node( + `${parent.id}/${frameId}`, + name, + location, + frame.in_app === true, + ); + childMap.set(frameId, child); + parent.children.push(child); + } + child.samples++; + child.totalMs += weight; + parent = child; + const key = `${name}:${location}`; + const fn = functions.get(key) ?? { + ...node(key, name, location, frame.in_app === true), + }; + if (!seenFunctions.has(key)) { + fn.samples++; + fn.totalMs += weight; + seenFunctions.add(key); + } + functions.set(key, fn); + } + parent.selfSamples++; + parent.selfMs += weight; + if (parent !== root) { + const fn = functions.get(`${parent.name}:${parent.location}`); + if (fn) { + fn.selfSamples++; + fn.selfMs += weight; + } + } + } + return { + root, + functions: [...functions.values()].sort( + (a, b) => b.totalMs - a.totalMs || b.samples - a.samples, + ), + sampleCount: selected.length, + durationMs: hasTiming ? root.totalMs : undefined, + threads, + threadId, + }; +} diff --git a/packages/telemetry/src/telemetry/public.ts b/packages/telemetry/src/telemetry/public.ts new file mode 100644 index 000000000..8276782d1 --- /dev/null +++ b/packages/telemetry/src/telemetry/public.ts @@ -0,0 +1,111 @@ +import type { Envelope, EnvelopeItem } from "@sentry/core"; +import type { + BatchOptions, + ChangeMetadata, + Diagnostic, + IngestOptions, + IngestResult, + IngestStats, +} from "./contracts.ts"; +import type { TelemetryReader } from "./reader.ts"; +import type { + ProfileFunctionSummary as InternalProfileFunctionSummary, + QuerySummary as InternalQuerySummary, + ResourceSummary as InternalResourceSummary, +} from "./selectors/index.ts"; + +import type { + AITrace as InternalAITrace, + Sdk, + SentryErrorEvent, + SentryEvent, + SentryLogEventItem, + SentryProfileWithTraceMeta, + Span, + Trace, +} from "./types.ts"; + +export type DeepReadonly = T extends (...args: never[]) => unknown + ? T + : T extends ReadonlyMap + ? ReadonlyMap, DeepReadonly> + : T extends ReadonlySet + ? ReadonlySet> + : T extends readonly unknown[] + ? { readonly [K in keyof T]: DeepReadonly } + : T extends object + ? { readonly [K in keyof T]: DeepReadonly } + : T; + +/** A parsed Sentry wire envelope; input timestamps retain their wire units. */ +// Item payloads remain unknown: forward-compatible wire records include span-v2 +// containers that older @sentry/core payload unions cannot represent. +export type SentryEnvelope = [Envelope[0], [EnvelopeItem[0], unknown][]]; +export type ProcessedEvent = DeepReadonly; +export type ProcessedSpan = DeepReadonly; +export type ProcessedTrace = DeepReadonly; +export type ProcessedLog = DeepReadonly; +export type ProcessedProfile = DeepReadonly; +export type AITrace = DeepReadonly; +export type QuerySummary = DeepReadonly; +export type ResourceSummary = DeepReadonly; +export type ProfileFunctionSummary = + DeepReadonly; +export type * from "./contracts.ts"; +export type { + TraceReadFilter as TraceFilter, + EventReadFilter as EventFilter, + DatabaseReadFilter as DatabaseQueryFilter, +} from "./reader.ts"; +export type RetentionOptions = { + /** Maximum retained envelopes, including all telemetry derived from them. Default: 500. */ + maxEnvelopes?: number; + /** Optional maximum age since receipt. Expiration runs automatically; dispose releases the timer. */ + maxAgeMs?: number; + /** Serialized retained-envelope budget. Default: 64 MiB. */ + maxBytes?: number; +}; +export type TelemetryStoreOptions = { + retention?: RetentionOptions; + enrichStacktrace?: ( + event: SentryErrorEvent, + context: { signal: AbortSignal }, + ) => Promise; + /** Maximum ingestion operations, including active work. Default: 64. Overflow returns ignored/queue_full. */ + maxPending?: number; + /** Maximum serialized parsed envelope size. Default: 32 MiB. */ + maxEnvelopeBytes?: number; + /** Optional enrichment deadline. Default: 5000 ms. */ + enrichmentTimeoutMs?: number; + onDiagnostic?: (diagnostic: Diagnostic) => void; +}; +export type TelemetrySnapshot = { + readonly revision: number; + readonly envelopes: readonly DeepReadonly[]; + readonly events: readonly ProcessedEvent[]; + readonly traces: readonly ProcessedTrace[]; + readonly logs: readonly ProcessedLog[]; + readonly sdks: readonly DeepReadonly[]; + readonly profiles: ReadonlyMap; +}; +export type TelemetryStore = TelemetryReader & { + ingestEnvelope: ( + input: unknown, + options?: IngestOptions, + ) => Promise; + ingestEnvelopes: ( + inputs: Iterable, + options?: BatchOptions, + ) => Promise; + /** Wait for operations submitted before this call; future operations are excluded. */ + drain: () => Promise; + getStats: () => IngestStats; + subscribe: ( + listener: (snapshot: TelemetrySnapshot, change: ChangeMetadata) => void, + ) => () => void; + reset: () => void; + /** Update supplied limits, preserving omitted fields, then wait for retention eviction. */ + setRetention: (options: RetentionOptions) => Promise; + prune: () => Promise; + dispose: () => void; +}; diff --git a/packages/telemetry/src/telemetry/queries.ts b/packages/telemetry/src/telemetry/queries.ts new file mode 100644 index 000000000..5684ae41c --- /dev/null +++ b/packages/telemetry/src/telemetry/queries.ts @@ -0,0 +1,2 @@ +// Internal compatibility barrel. Prefer importing from the individual selector domain. +export * from "./selectors/index.ts"; diff --git a/packages/telemetry/src/telemetry/reader.spec.ts b/packages/telemetry/src/telemetry/reader.spec.ts new file mode 100644 index 000000000..b275560ca --- /dev/null +++ b/packages/telemetry/src/telemetry/reader.spec.ts @@ -0,0 +1,281 @@ +// @vitest-environment node +import { expect, test } from "vitest"; +import type { TelemetrySnapshot } from "./public.ts"; +import { createTelemetryReader } from "./reader.ts"; +import type { Span, Trace } from "./types.ts"; +function fixture() { + const spans: Span[] = [ + { + span_id: "b", + start_timestamp: 10, + timestamp: 20, + op: "db.query", + description: "SELECT 1", + }, + { + span_id: "a", + start_timestamp: 5, + timestamp: 20, + op: "db.query", + description: "SELECT 1", + }, + { + span_id: "c", + start_timestamp: 25, + timestamp: 30, + op: "resource.script", + description: "script.js", + data: { "http.response_content_length": 40 }, + }, + ]; + const trace: Trace = { + trace_id: "trace", + transactions: [], + errors: 0, + start_timestamp: 5, + timestamp: 30, + rootTransaction: null, + rootTransactionName: "", + spans: new Map(spans.map((span) => [span.span_id, span])), + spanTree: spans, + profileGrafted: false, + }; + const snapshot: TelemetrySnapshot = { + revision: 1, + envelopes: [], + events: [ + { + event_id: "b", + timestamp: 20, + exception: { value: undefined, values: [] }, + }, + { + event_id: "a", + timestamp: 20, + exception: { value: undefined, values: [] }, + }, + { + event_id: "c", + timestamp: 30, + exception: { value: undefined, values: [] }, + }, + ], + traces: [trace], + logs: [], + sdks: [], + profiles: new Map(), + }; + return { snapshot, spans }; +} +test("inclusive milliseconds and deterministic IDs apply without truncating defaults", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + expect( + reader.getEvents({ fromMs: 20, toMs: 20 }).map((event) => event.event_id), + ).toEqual(["a", "b"]); + expect( + reader + .getEvents({ order: "desc", limit: 2 }) + .map((event) => event.event_id), + ).toEqual(["c", "b"]); + expect( + reader + .getSpans({ traceId: "trace", fromMs: 20, toMs: 20 }) + .map((span) => span.span_id), + ).toEqual(["a", "b"]); + expect(reader.getSpans({ traceId: "other" })).toEqual([]); + const many = { + ...snapshot, + events: Array.from({ length: 1200 }, (_, n) => ({ + event_id: String(n), + timestamp: n, + exception: { value: undefined, values: [] }, + })), + }; + expect(createTelemetryReader(() => many).getEvents()).toHaveLength(1200); +}); +test("invalid getter filters throw", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + for (const filter of [ + { fromMs: Number.NaN }, + { fromMs: 20, toMs: 10 }, + { limit: 0 }, + { limit: 1.5 }, + { order: "sideways" }, + ]) + expect(() => + reader.getEvents(filter as Parameters[0]), + ).toThrow(RangeError); +}); +test("summaries default to compact and preserve trace-less legacy spans", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + expect( + reader.getDatabaseQuerySummaries({ traceId: "trace", toMs: 20 }), + ).toEqual([ + { + description: "SELECT 1", + count: 2, + totalDurationMs: 25, + averageDurationMs: 12.5, + }, + ]); + expect( + reader.getDatabaseQuerySummaries({ includeSpans: true })[0].spans, + ).toHaveLength(2); + expect(reader.getResourceSummaries()).toEqual([ + { + description: "script.js", + count: 1, + totalDurationMs: 5, + averageDurationMs: 5, + averageEncodedSizeBytes: 40, + averageSizeBytes: 40, + }, + ]); + expect( + reader.getResourceSummaries({ includeSpans: true })[0].spans, + ).toHaveLength(1); +}); +test("canonical filters share cached identity until revision changes", () => { + let { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + const first = reader.getDatabaseQuerySummaries({ + traceId: "trace", + fromMs: 10, + }); + expect( + reader.getDatabaseQuerySummaries({ fromMs: 10, traceId: "trace" }), + ).toBe(first); + expect(reader.getEvents()).toBe(reader.getEvents({})); + snapshot = { ...snapshot, revision: 2 }; + expect( + reader.getDatabaseQuerySummaries({ fromMs: 10, traceId: "trace" }), + ).not.toBe(first); +}); +test("page cursors bind collection, canonical query and revision", () => { + let { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + const first = reader.readPage("events", { limit: 1, fromMs: 20 }); + expect(first).toMatchObject({ + status: "ok", + revision: 1, + total: 3, + items: [{ event_id: "a" }], + }); + if (first.status !== "ok") throw new Error("expected page"); + expect( + reader.readPage("events", { + fromMs: 20, + limit: 1, + cursor: first.nextCursor, + }), + ).toMatchObject({ + status: "ok", + items: [{ event_id: "b" }], + }); + expect( + reader.readPage("logs", { fromMs: 20, limit: 1, cursor: first.nextCursor }), + ).toMatchObject({ + status: "invalid", + code: "invalid_cursor", + }); + expect( + reader.readPage("events", { + fromMs: 21, + limit: 1, + cursor: first.nextCursor, + }), + ).toMatchObject({ + status: "invalid", + code: "invalid_cursor", + }); + snapshot = { ...snapshot, revision: 2 }; + expect( + reader.readPage("events", { + fromMs: 20, + limit: 1, + cursor: first.nextCursor, + }), + ).toMatchObject({ + status: "invalid", + code: "stale_cursor", + }); + expect(reader.readPage("events", { cursor: "bad" })).toMatchObject({ + status: "invalid", + code: "invalid_cursor", + }); + expect(reader.readPage("events", { limit: 1001 })).toMatchObject({ + status: "invalid", + code: "invalid_query", + }); + expect(reader.readPage("events", { fromMs: 30, toMs: 20 })).toMatchObject({ + status: "invalid", + code: "invalid_query", + }); +}); +test("pages default to 100 and memoize result", () => { + const { snapshot } = fixture(); + const many = { + ...snapshot, + events: Array.from({ length: 1200 }, (_, n) => ({ + event_id: String(n), + timestamp: n, + exception: { value: undefined, values: [] }, + })), + }; + const reader = createTelemetryReader(() => many); + const page = reader.readPage("events"); + expect(page.status === "ok" && page.items.length).toBe(100); + expect(reader.readPage("events", {})).toBe(page); +}); + +test("explicit default filter values share cached results", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + expect(reader.getEvents({ order: "asc" })).toBe(reader.getEvents()); + expect(reader.getDatabaseQuerySummaries({ includeSpans: false })).toBe( + reader.getDatabaseQuerySummaries(), + ); +}); + +test("web vitals apply limit after selecting measurements", () => { + const { snapshot } = fixture(); + const measured = { + event_id: "vital", + timestamp: 40, + type: "transaction" as const, + start_timestamp: 30, + contexts: { trace: { trace_id: "trace", op: "pageload" } }, + measurements: { + fcp: { value: 100, unit: "millisecond" }, + lcp: { value: 100, unit: "millisecond" }, + cls: { value: 0, unit: "" }, + ttfb: { value: 50, unit: "millisecond" }, + fid: { value: 10, unit: "millisecond" }, + }, + }; + const reader = createTelemetryReader(() => ({ + ...snapshot, + events: [...snapshot.events, measured], + })); + expect( + reader.getWebVitals({ limit: 1 }).map((event) => event.event_id), + ).toEqual(["vital"]); +}); + +test("reader bounds cached queries while retaining recent identities", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + const oldest = reader.getEvents({ fromMs: -1 }); + const recent = reader.getEvents({ fromMs: 0 }); + for (let index = 1; index <= 260; index++) { + reader.getEvents({ fromMs: index }); + expect(reader.getEvents({ fromMs: 0 })).toBe(recent); + } + expect(reader.getEvents({ fromMs: -1 })).not.toBe(oldest); + expect(reader.readPage("events", { limit: 0 })).toMatchObject({ + revision: 1, + }); +}); diff --git a/packages/telemetry/src/telemetry/reader.ts b/packages/telemetry/src/telemetry/reader.ts new file mode 100644 index 000000000..68ef61ca0 --- /dev/null +++ b/packages/telemetry/src/telemetry/reader.ts @@ -0,0 +1,543 @@ +import { readonlyCopy } from "./internal/readonly.ts"; +import type { + AITrace, + DeepReadonly, + ProcessedEvent, + ProcessedLog, + ProcessedProfile, + ProcessedSpan, + ProcessedTrace, + ProfileFunctionSummary, + QuerySummary, + ResourceSummary, + TelemetrySnapshot, +} from "./public.ts"; +import { + getAITraces, + getDatabaseQuerySummaries, + getProfileFunctionSummaries, + getResourceSummaries, + getWebVitals, +} from "./selectors/index.ts"; +import type { + Sdk, + SentryErrorEvent, + SentryEvent, + SentryEventWithPerformanceData, + Span, + Trace, +} from "./types.ts"; +import { isErrorEvent } from "./utils/sentry.ts"; + +/** Inclusive bounds in Unix milliseconds. No implicit limit is applied to getters. */ +export type TelemetryReadFilter = { + readonly fromMs?: number; + readonly toMs?: number; + readonly order?: "asc" | "desc"; + readonly limit?: number; +}; +export type TraceReadFilter = TelemetryReadFilter & { + readonly traceId?: string; +}; +export type EventReadFilter = TraceReadFilter & { readonly spanId?: string }; +export type SummaryReadFilter = TraceReadFilter & { + readonly includeSpans?: boolean; +}; +export type DatabaseReadFilter = SummaryReadFilter & { + readonly description?: string; +}; +export type CompactQuerySummary = Omit; +export type CompactResourceSummary = Omit; +export type ReadCollection = "events" | "traces" | "logs" | "spans"; +export type ReadPageOptions = EventReadFilter & { readonly cursor?: string }; +export type ReadCollectionItem = { + events: ProcessedEvent; + traces: ProcessedTrace; + logs: ProcessedLog; + spans: ProcessedSpan; +}[C]; +export type ReadPageResult = + | { + readonly status: "ok"; + readonly revision: number; + readonly items: readonly T[]; + readonly total: number; + readonly nextCursor?: string; + } + | { + readonly status: "invalid"; + readonly revision: number; + readonly code: "stale_cursor" | "invalid_cursor" | "invalid_query"; + readonly message: string; + }; +export type TelemetryReader = { + getSnapshot(): TelemetrySnapshot; + getEnvelope(id: string): TelemetrySnapshot["envelopes"][number] | undefined; + getEnvelopes(): TelemetrySnapshot["envelopes"]; + getEvent(id: string): ProcessedEvent | undefined; + getEvents(filter?: EventReadFilter): readonly ProcessedEvent[]; + getErrors( + filter?: EventReadFilter, + ): readonly DeepReadonly[]; + getTrace(id: string): ProcessedTrace | undefined; + getTraces(filter?: TraceReadFilter): readonly ProcessedTrace[]; + getSpans(filter?: TraceReadFilter): readonly ProcessedSpan[]; + getLog(id: string): ProcessedLog | undefined; + getLogs(filter?: TraceReadFilter): readonly ProcessedLog[]; + getProfile(traceId: string): ProcessedProfile | undefined; + getProfileFunctionSummaries( + filter?: TraceReadFilter, + ): readonly ProfileFunctionSummary[]; + getDatabaseQuerySummaries( + filter: DatabaseReadFilter & { includeSpans: true }, + ): readonly QuerySummary[]; + getDatabaseQuerySummaries( + filter?: DatabaseReadFilter & { includeSpans?: false }, + ): readonly CompactQuerySummary[]; + getDatabaseQuerySummaries( + filter?: DatabaseReadFilter, + ): readonly (CompactQuerySummary & { + readonly spans?: QuerySummary["spans"]; + })[]; + getResourceSummaries( + filter: SummaryReadFilter & { includeSpans: true }, + ): readonly ResourceSummary[]; + getResourceSummaries( + filter?: SummaryReadFilter & { includeSpans?: false }, + ): readonly CompactResourceSummary[]; + getResourceSummaries( + filter?: SummaryReadFilter, + ): readonly (CompactResourceSummary & { + readonly spans?: ResourceSummary["spans"]; + })[]; + getAITraces(filter?: TraceReadFilter): readonly AITrace[]; + getWebVitals( + filter?: EventReadFilter, + ): readonly DeepReadonly[]; + getSdks(): readonly DeepReadonly[]; + readPage( + collection: C, + options?: ReadPageOptions, + ): ReadPageResult>; +}; + +function validate(filter: ReadPageOptions, paged = false) { + for (const value of [filter.fromMs, filter.toMs]) + if (value !== undefined && !Number.isFinite(value)) + throw new RangeError("Time bounds must be finite Unix milliseconds"); + if ( + filter.fromMs !== undefined && + filter.toMs !== undefined && + filter.fromMs > filter.toMs + ) + throw new RangeError("fromMs must not exceed toMs"); + if ( + filter.order !== undefined && + filter.order !== "asc" && + filter.order !== "desc" + ) + throw new RangeError("order must be asc or desc"); + if ( + filter.limit !== undefined && + (!Number.isSafeInteger(filter.limit) || + filter.limit < 1 || + (paged && filter.limit > 1000)) + ) + throw new RangeError( + paged + ? "Page limit must be between 1 and 1000" + : "limit must be a positive safe integer", + ); + for (const value of [filter.traceId, filter.spanId]) + if (value !== undefined && typeof value !== "string") + throw new RangeError("IDs must be strings"); +} +function canonical(value: object): string { + return JSON.stringify( + Object.fromEntries( + Object.entries(value) + .filter( + ([key, item]) => + item !== undefined && + !(key === "order" && item === "asc") && + !(key === "includeSpans" && item === false), + ) + .sort(([a], [b]) => a.localeCompare(b)), + ), + ); +} +function within(timestamp: number, filter: TelemetryReadFilter) { + return ( + (filter.fromMs === undefined || timestamp >= filter.fromMs) && + (filter.toMs === undefined || timestamp <= filter.toMs) + ); +} +function ordered( + items: readonly T[], + filter: TelemetryReadFilter, + time: (item: T) => number, + id: (item: T) => string, +): readonly T[] { + const direction = filter.order === "desc" ? -1 : 1; + return Object.freeze( + items + .filter((item) => within(time(item), filter)) + .sort( + (a, b) => + direction * + (time(a) - time(b) || (id(a) < id(b) ? -1 : id(a) > id(b) ? 1 : 0)), + ) + .slice(0, filter.limit), + ); +} +function named( + items: readonly T[], + filter: TelemetryReadFilter, + name: (item: T) => string, +): readonly T[] { + return ordered( + items, + { order: filter.order, limit: filter.limit }, + () => 0, + name, + ); +} + +/** A portable read-only dependency. The current revision retains at most 256 recently used reads. */ +export function createTelemetryReader( + getSnapshot: () => TelemetrySnapshot, +): TelemetryReader { + let revision = -1; + const cache = new Map(); + function read( + name: string, + filter: object, + derive: (snapshot: TelemetrySnapshot) => T, + ): T { + const snapshot = getSnapshot(); + if (revision !== snapshot.revision) { + cache.clear(); + revision = snapshot.revision; + } + const key = `${name}:${canonical(filter)}`; + if (cache.has(key)) { + const value = cache.get(key) as T; + cache.delete(key); + cache.set(key, value); + return value; + } + const value = derive(snapshot); + cache.set(key, value); + if (cache.size > 256) cache.delete(cache.keys().next().value!); + return value; + } + const traces = (snapshot: TelemetrySnapshot, filter: TraceReadFilter) => + snapshot.traces.filter( + (trace) => + filter.traceId === undefined || trace.trace_id === filter.traceId, + ); + // Legacy transaction child spans need not carry trace_id. Scope their owning trace first. + const spans = (snapshot: TelemetrySnapshot, filter: TraceReadFilter) => + traces(snapshot, filter).flatMap((trace) => + Array.from(trace.spans.values()), + ); + function events(snapshot: TelemetrySnapshot, filter: EventReadFilter) { + return snapshot.events.filter( + (event) => + (filter.traceId === undefined || + event.contexts?.trace?.trace_id === filter.traceId || + (event.type === "span" && event.span.trace_id === filter.traceId)) && + (filter.spanId === undefined || + event.contexts?.trace?.span_id === filter.spanId || + (event.type === "span" && event.span.span_id === filter.spanId)), + ); + } + const reader: TelemetryReader = { + getSnapshot, + getEnvelope: (id) => + getSnapshot().envelopes.find( + (envelope) => String(envelope[0].__spotlight_envelope_id) === id, + ), + getEnvelopes: () => getSnapshot().envelopes, + getEvent: (id) => + getSnapshot().events.find((event) => event.event_id === id), + getEvents(filter = {}) { + validate(filter); + return read("events", filter, (snapshot) => + ordered( + events(snapshot, filter), + filter, + (event) => event.timestamp, + (event) => event.event_id, + ), + ); + }, + getErrors(filter = {}) { + validate(filter); + return read("errors", filter, (snapshot) => + ordered( + events(snapshot, filter).filter((event) => + isErrorEvent(event as SentryEvent), + ) as readonly DeepReadonly[], + filter, + (event) => event.timestamp, + (event) => event.event_id, + ), + ); + }, + getTrace: (id) => + getSnapshot().traces.find((trace) => trace.trace_id === id), + getTraces(filter = {}) { + validate(filter); + return read("traces", filter, (snapshot) => + ordered( + traces(snapshot, filter), + filter, + (trace) => trace.timestamp, + (trace) => trace.trace_id, + ), + ); + }, + getSpans(filter = {}) { + validate(filter); + return read("spans", filter, (snapshot) => + ordered( + spans(snapshot, filter), + filter, + (span) => span.timestamp, + (span) => span.span_id, + ), + ); + }, + getLog: (id) => getSnapshot().logs.find((log) => log.id === id), + getLogs(filter = {}) { + validate(filter); + return read("logs", filter, (snapshot) => + ordered( + snapshot.logs.filter( + (log) => + filter.traceId === undefined || log.trace_id === filter.traceId, + ), + filter, + (log) => log.timestamp, + (log) => log.id, + ), + ); + }, + getProfile: (id) => getSnapshot().profiles.get(id), + getProfileFunctionSummaries(filter = {}) { + validate(filter); + return read("profiles", filter, (snapshot) => + readonlyCopy( + named( + getProfileFunctionSummaries( + new Map( + Array.from(snapshot.profiles).filter( + ([id, profile]) => + (filter.traceId === undefined || id === filter.traceId) && + within(profile.timestamp, filter), + ), + ) as unknown as Parameters[0], + ), + filter, + (summary) => summary.name, + ), + ), + ); + }, + getDatabaseQuerySummaries: ((input?: DatabaseReadFilter) => { + const filter = input ?? {}; + validate(filter); + if ( + filter.description !== undefined && + typeof filter.description !== "string" + ) + throw new RangeError("description must be a string"); + if ( + filter.includeSpans !== undefined && + typeof filter.includeSpans !== "boolean" + ) + throw new RangeError("includeSpans must be boolean"); + return read("queries", filter, (snapshot) => + readonlyCopy( + named( + getDatabaseQuerySummaries( + ordered( + spans(snapshot, filter), + { + fromMs: filter.fromMs, + toMs: filter.toMs, + order: filter.order, + }, + (span) => span.timestamp, + (span) => span.span_id, + ) as Span[], + { description: filter.description }, + ).map((summary) => { + const { spans: detail, ...compact } = summary; + return filter.includeSpans + ? { ...compact, spans: detail } + : compact; + }), + filter, + (summary) => summary.description, + ), + ), + ); + }) as TelemetryReader["getDatabaseQuerySummaries"], + getResourceSummaries: ((input?: SummaryReadFilter) => { + const filter = input ?? {}; + validate(filter); + if ( + filter.includeSpans !== undefined && + typeof filter.includeSpans !== "boolean" + ) + throw new RangeError("includeSpans must be boolean"); + return read("resources", filter, (snapshot) => + readonlyCopy( + named( + getResourceSummaries( + ordered( + spans(snapshot, filter), + { + fromMs: filter.fromMs, + toMs: filter.toMs, + order: filter.order, + }, + (span) => span.timestamp, + (span) => span.span_id, + ) as Span[], + ).map((summary) => { + const { spans: detail, ...compact } = summary; + return filter.includeSpans + ? { ...compact, spans: detail } + : compact; + }), + filter, + (summary) => summary.description, + ), + ), + ); + }) as TelemetryReader["getResourceSummaries"], + getAITraces(filter = {}) { + validate(filter); + return read("ai", filter, (snapshot) => + readonlyCopy( + ordered( + getAITraces(traces(snapshot, filter) as Trace[]), + filter, + (trace) => trace.timestamp, + (trace) => trace.id, + ), + ), + ); + }, + getWebVitals(filter = {}) { + validate(filter); + return read("vitals", filter, (snapshot) => + readonlyCopy( + ordered( + getWebVitals(events(snapshot, filter) as SentryEvent[]), + filter, + (event) => event.timestamp, + (event) => event.event_id, + ), + ), + ); + }, + getSdks: () => getSnapshot().sdks, + readPage(collection, options = {}) { + const invalid = ( + code: "invalid_query" | "invalid_cursor" | "stale_cursor", + message: string, + ) => + Object.freeze({ + status: "invalid" as const, + revision: getSnapshot().revision, + code, + message, + }); + try { + validate(options, true); + if (!["events", "traces", "logs", "spans"].includes(collection)) + throw new RangeError("Unknown collection"); + if (collection !== "events" && options.spanId !== undefined) + throw new RangeError("spanId is only supported for events"); + } catch (error) { + return invalid( + "invalid_query", + error instanceof Error ? error.message : "Invalid query", + ); + } + const { cursor, ...filter } = options; + const query = canonical({ + ...filter, + order: filter.order ?? "asc", + limit: filter.limit ?? 100, + }); + const snapshot = getSnapshot(); + let offset = 0; + if (cursor !== undefined) { + try { + if (typeof cursor !== "string" || cursor.length > 16384) + throw new Error(); + const parsed = JSON.parse(decodeURIComponent(cursor)); + if ( + parsed.version !== 1 || + !Number.isSafeInteger(parsed.revision) || + !Number.isSafeInteger(parsed.offset) || + parsed.offset < 1 || + parsed.collection !== collection || + parsed.query !== query + ) + throw new Error(); + if (parsed.revision !== snapshot.revision) + return invalid( + "stale_cursor", + "The snapshot changed; restart pagination", + ); + offset = parsed.offset; + } catch { + return invalid("invalid_cursor", "Cursor does not match this query"); + } + } + return read(`page:${collection}`, { query, cursor }, () => { + const { limit: _limit, ...allFilter } = filter; + const items = + collection === "events" + ? reader.getEvents(allFilter) + : collection === "traces" + ? reader.getTraces(allFilter) + : collection === "logs" + ? reader.getLogs(allFilter) + : reader.getSpans(allFilter); + if (offset > items.length || offset % (filter.limit ?? 100) !== 0) + return invalid( + "invalid_cursor", + "Cursor offset is outside this query", + ); + const end = offset + (filter.limit ?? 100); + return Object.freeze({ + status: "ok" as const, + revision: snapshot.revision, + items: Object.freeze(items.slice(offset, end)), + total: items.length, + ...(end < items.length + ? { + nextCursor: encodeURIComponent( + JSON.stringify({ + version: 1, + revision: snapshot.revision, + collection, + query, + offset: end, + }), + ), + } + : {}), + }); + }) as ReadPageResult>; + }, + }; + return Object.freeze(reader); +} diff --git a/packages/telemetry/src/telemetry/selectors/ai.ts b/packages/telemetry/src/telemetry/selectors/ai.ts new file mode 100644 index 000000000..b2942860b --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/ai.ts @@ -0,0 +1,18 @@ +import { + createAITraceFromSpan, + extractAllAIRootSpans, +} from "../ai/aiLibraries.ts"; +import type { AITrace, Span, Trace } from "../types.ts"; + +export function getAIRootSpans(traces: Iterable): Span[] { + return Array.from(traces).flatMap((trace) => + extractAllAIRootSpans(trace.spanTree).map(({ span }) => span), + ); +} + +export function getAITraces(traces: Iterable): AITrace[] { + return getAIRootSpans(traces).flatMap((span) => { + const result = createAITraceFromSpan(span); + return result ? [result] : []; + }); +} diff --git a/packages/telemetry/src/telemetry/selectors/database.ts b/packages/telemetry/src/telemetry/selectors/database.ts new file mode 100644 index 000000000..d859c28aa --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/database.ts @@ -0,0 +1,24 @@ +import type { Span } from "../types.ts"; +import { type SpanFilter, type SpanSummary, summarizeSpans } from "./spans.ts"; + +export const DB_SPAN_REGEX = /^db(?:\.[A-Za-z]+)*$/; +export type QueryFilter = SpanFilter & { description?: string }; +export type QuerySummary = SpanSummary; + +/** Inputs and durations use normalized milliseconds. Presentation sorting belongs to callers. */ +export function getDatabaseQuerySummaries( + spans: readonly Span[], + filter: QueryFilter = {}, +): QuerySummary[] { + return summarizeSpans( + spans.filter( + (span) => + DB_SPAN_REGEX.test(span.op ?? "") && + (filter.traceId === undefined || span.trace_id === filter.traceId), + ), + ).filter( + (query) => + filter.description === undefined || + query.description === filter.description.trim(), + ); +} diff --git a/packages/telemetry/src/telemetry/selectors/index.ts b/packages/telemetry/src/telemetry/selectors/index.ts new file mode 100644 index 000000000..ee4b292f9 --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/index.ts @@ -0,0 +1,14 @@ +export { getAIRootSpans, getAITraces } from "./ai.ts"; +export { + DB_SPAN_REGEX, + getDatabaseQuerySummaries, + type QueryFilter, + type QuerySummary, +} from "./database.ts"; +export { + getProfileFunctionSummaries, + type ProfileFunctionSummary, +} from "./profiles.ts"; +export { getResourceSummaries, type ResourceSummary } from "./resources.ts"; +export { getSpans, type SpanFilter } from "./spans.ts"; +export { getWebVitals } from "./webVitals.ts"; diff --git a/packages/telemetry/src/telemetry/selectors/profiles.ts b/packages/telemetry/src/telemetry/selectors/profiles.ts new file mode 100644 index 000000000..ef77a599a --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/profiles.ts @@ -0,0 +1,48 @@ +import type { SentryProcessedProfile, TraceId } from "../types.ts"; +import { getFunctionNameFromFrame } from "../utils/profileUtils.ts"; +import type { SpanFilter } from "./spans.ts"; + +export type ProfileFunctionSummary = { + name: string; + totalDurationMs: number; + sampleCount: number; + traceIds: Set; +}; + +/** Inclusive frame durations in milliseconds, using each sample's interval to the next sample. */ +export function getProfileFunctionSummaries( + profilesByTraceId: ReadonlyMap, + filter: SpanFilter = {}, +): ProfileFunctionSummary[] { + const aggregateCalls = new Map(); + for (const [traceId, profile] of profilesByTraceId) { + if (filter.traceId !== undefined && traceId !== filter.traceId) continue; + // The final sample has no known duration and is not included. + for ( + let sampleIdx = 0; + sampleIdx < profile.samples.length - 1; + sampleIdx++ + ) { + const sample = profile.samples[sampleIdx]; + const duration = + profile.samples[sampleIdx + 1].start_timestamp - sample.start_timestamp; + for (const frameIdx of profile.stacks[sample.stack_id]) { + const name = getFunctionNameFromFrame(profile.frames[frameIdx]); + const callData = aggregateCalls.get(name); + if (callData) { + callData.totalDurationMs += duration; + callData.sampleCount += 1; + callData.traceIds.add(traceId); + } else { + aggregateCalls.set(name, { + name, + totalDurationMs: duration, + sampleCount: 1, + traceIds: new Set([traceId]), + }); + } + } + } + } + return Array.from(aggregateCalls.values()); +} diff --git a/packages/telemetry/src/telemetry/selectors/resources.ts b/packages/telemetry/src/telemetry/selectors/resources.ts new file mode 100644 index 000000000..68d664b24 --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/resources.ts @@ -0,0 +1,53 @@ +import type { Span } from "../types.ts"; +import { type SpanSummary, summarizeSpans } from "./spans.ts"; + +export type ResourceSummary = SpanSummary & { + averageEncodedSizeBytes: number; + /** Best available captured response size. Unknown sizes are excluded. */ + averageSizeBytes?: number; +}; + +function knownResourceSize(span: Span): number | undefined { + for (const key of [ + "http.response_content_length", + "http.response.body.size", + "http.decoded_response_content_length", + ]) { + const value = span.data?.[key]; + if (typeof value === "number" && Number.isFinite(value) && value >= 0) + return value; + } + return undefined; +} + +export function getResourceSummaries( + spans: readonly Span[], +): ResourceSummary[] { + return summarizeSpans( + spans.filter((span) => /resource\.[A-Za-z]+/.test(span.op ?? "")), + ).map((group) => { + const sizes = group.spans.flatMap((span) => { + const size = knownResourceSize(span); + return size === undefined ? [] : [size]; + }); + return { + ...group, + averageEncodedSizeBytes: + group.spans.reduce((sum, span) => { + const size = span.data?.["http.response_content_length"]; + return ( + sum + + (typeof size === "number" && Number.isFinite(size) && size >= 0 + ? size + : 0) + ); + }, 0) / group.count, + ...(sizes.length + ? { + averageSizeBytes: + sizes.reduce((sum, size) => sum + size, 0) / sizes.length, + } + : {}), + }; + }); +} diff --git a/packages/telemetry/src/telemetry/selectors/selectors.test.ts b/packages/telemetry/src/telemetry/selectors/selectors.test.ts new file mode 100644 index 000000000..c99605b9a --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/selectors.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from "vitest"; +import type { SentryProcessedProfile, Span } from "../types.ts"; +import { getDatabaseQuerySummaries } from "./database.ts"; +import { getProfileFunctionSummaries } from "./profiles.ts"; +import { getResourceSummaries } from "./resources.ts"; + +function span(overrides: Partial = {}): Span { + return { + span_id: "span-id", + trace_id: "trace-a", + op: "db.sql.query", + description: " SELECT 1 ", + start_timestamp: 1000, + timestamp: 1010, + ...overrides, + }; +} + +function profile(): SentryProcessedProfile { + return { + frames: [ + { function: "main", module: "app" }, + { function: "query", module: "app" }, + { function: "final", module: "app" }, + ], + stacks: [[0, 1], [0], [2]], + samples: [ + { start_timestamp: 1000, stack_id: 0, thread_id: "main" }, + { start_timestamp: 1010, stack_id: 1, thread_id: "main" }, + { start_timestamp: 1040, stack_id: 2, thread_id: "main" }, + ], + }; +} + +describe("domain summaries", () => { + it("groups database spans using normalized millisecond fields and excludes invalid inputs", () => { + const first = span(); + const second = span({ + span_id: "second", + description: "SELECT 1", + timestamp: 1030, + }); + const result = getDatabaseQuerySummaries([ + first, + span({ op: "http.client" }), + second, + span({ description: " " }), + span({ timestamp: Number.NaN }), + span({ start_timestamp: Number.POSITIVE_INFINITY }), + ]); + + expect(result).toEqual([ + { + description: "SELECT 1", + count: 2, + totalDurationMs: 40, + averageDurationMs: 20, + spans: [first, second], + }, + ]); + expect(first.description).toBe(" SELECT 1 "); + expect( + getDatabaseQuerySummaries([first], { description: " SELECT 1 " }), + ).toHaveLength(1); + expect( + getDatabaseQuerySummaries([first], { description: "SELECT 2" }), + ).toEqual([]); + expect(getDatabaseQuerySummaries([first], { traceId: "trace-b" })).toEqual( + [], + ); + }); + + it("reports resource size in bytes and retains member spans once", () => { + const first = span({ + op: "resource.script", + data: { "http.response_content_length": 400 }, + }); + const second = span({ op: "resource.script", timestamp: 1030 }); + const third = span({ + op: "resource.script", + data: { "http.response_content_length": "unknown" }, + }); + expect(getResourceSummaries([first, second, third, span()])).toEqual([ + { + description: "SELECT 1", + count: 3, + totalDurationMs: 50, + averageDurationMs: 50 / 3, + averageEncodedSizeBytes: 400 / 3, + averageSizeBytes: 400, + spans: [first, second, third], + }, + ]); + }); + + it("aggregates inclusive profile frame intervals across traces and excludes the unbounded final sample", () => { + const profiles = new Map([ + ["trace-a", profile()], + ["trace-b", profile()], + ]); + expect(getProfileFunctionSummaries(profiles)).toEqual([ + { + name: "app@main", + totalDurationMs: 80, + sampleCount: 4, + traceIds: new Set(["trace-a", "trace-b"]), + }, + { + name: "app@query", + totalDurationMs: 20, + sampleCount: 2, + traceIds: new Set(["trace-a", "trace-b"]), + }, + ]); + expect( + getProfileFunctionSummaries(profiles, { traceId: "trace-a" }), + ).toEqual([ + { + name: "app@main", + totalDurationMs: 40, + sampleCount: 2, + traceIds: new Set(["trace-a"]), + }, + { + name: "app@query", + totalDurationMs: 10, + sampleCount: 1, + traceIds: new Set(["trace-a"]), + }, + ]); + expect( + getProfileFunctionSummaries(profiles, { traceId: "missing" }), + ).toEqual([]); + expect( + getProfileFunctionSummaries( + new Map([["empty", { ...profile(), samples: [] }]]), + ), + ).toEqual([]); + expect(profiles.get("trace-a")?.samples).toHaveLength(3); + }); +}); + +it("averages known resource sizes across captured aliases without treating absent or invalid sizes as zero", () => { + const sizes = [ + { "http.response_content_length": 400 }, + { "http.response.body.size": 200 }, + { "http.decoded_response_content_length": 300 }, + { + "http.response_content_length": "invalid", + "http.response.body.size": 100, + }, + {}, + { "http.response.body.size": -1 }, + { "http.response.body.size": Number.NaN }, + { "http.response.body.size": 0 }, + ]; + const result = getResourceSummaries( + sizes.map((data, index) => + span({ span_id: String(index), op: "resource.script", data }), + ), + ); + expect(result[0]).toMatchObject({ + count: 8, + averageEncodedSizeBytes: 50, + averageSizeBytes: 200, + }); + expect( + getResourceSummaries([span({ op: "resource.script" })])[0], + ).not.toHaveProperty("averageSizeBytes"); +}); diff --git a/packages/telemetry/src/telemetry/selectors/spans.ts b/packages/telemetry/src/telemetry/selectors/spans.ts new file mode 100644 index 000000000..bb2b76496 --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/spans.ts @@ -0,0 +1,50 @@ +import type { Span, Trace } from "../types.ts"; + +export type SpanFilter = { traceId?: string }; + +export type SpanSummary = { + description: string; + count: number; + totalDurationMs: number; + averageDurationMs: number; + spans: Span[]; +}; + +export function getSpans( + traces: Iterable, + filter: SpanFilter = {}, +): Span[] { + return Array.from(traces) + .filter( + (trace) => + filter.traceId === undefined || trace.trace_id === filter.traceId, + ) + .flatMap((trace) => Array.from(trace.spans.values())); +} + +/** Groups normalized spans by description, preserving their input order. Durations are milliseconds. */ +export function summarizeSpans(spans: Iterable): SpanSummary[] { + const groups = new Map(); + for (const span of spans) { + const description = span.description?.trim(); + if ( + !description || + !Number.isFinite(span.timestamp) || + !Number.isFinite(span.start_timestamp) + ) + continue; + const group = groups.get(description) ?? { + description, + count: 0, + totalDurationMs: 0, + averageDurationMs: 0, + spans: [], + }; + group.count += 1; + group.totalDurationMs += span.timestamp - span.start_timestamp; + group.averageDurationMs = group.totalDurationMs / group.count; + group.spans.push(span); + groups.set(description, group); + } + return Array.from(groups.values()); +} diff --git a/packages/telemetry/src/telemetry/selectors/vitalPages.ts b/packages/telemetry/src/telemetry/selectors/vitalPages.ts new file mode 100644 index 000000000..ded6505e8 --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/vitalPages.ts @@ -0,0 +1,58 @@ +import { record, finite } from "../internal/wireValues.ts"; +export const WEB_VITALS = { + lcp: { label: "LCP", unit: "ms", good: 2500, poor: 4000 }, + inp: { label: "INP", unit: "ms", good: 200, poor: 500 }, + cls: { label: "CLS", unit: "", good: 0.1, poor: 0.25 }, + fcp: { label: "FCP", unit: "ms", good: 1800, poor: 3000 }, + ttfb: { label: "TTFB", unit: "ms", good: 800, poor: 1800 }, + fid: { label: "FID (legacy)", unit: "ms", good: 100, poor: 300 }, +} as const; +export type VitalName = keyof typeof WEB_VITALS; +export type VitalMeasurement = { + value: number; + rating: "Good" | "Needs improvement" | "Poor"; +}; +export type VitalVisit = { + eventId: string; + metrics: Partial>; +}; +export type VitalPage = { name: string; visits: VitalVisit[] }; +export type WebVitalEvent = { + id: string; + transaction?: string; + measurements?: unknown; +}; +export function groupWebVitalPages( + events: readonly WebVitalEvent[], +): VitalPage[] { + const pages = new Map(); + for (const event of events) { + const measurements = record(event.measurements); + if (!Object.keys(measurements).length) continue; + const metrics: VitalVisit["metrics"] = {}; + for (const name of Object.keys(WEB_VITALS) as VitalName[]) { + const metric = record(measurements[name]), + value = finite(metric.value); + if (value === undefined || value < 0) continue; + const config = WEB_VITALS[name]; + // SDK measurements normally use milliseconds; honor seconds when explicitly supplied. + const normalized = + config.unit === "ms" && metric.unit === "second" ? value * 1000 : value; + metrics[name] = { + value: normalized, + rating: + normalized <= config.good + ? "Good" + : normalized <= config.poor + ? "Needs improvement" + : "Poor", + }; + } + if (!Object.keys(metrics).length) continue; + const name = event.transaction ?? "Unknown page"; + const page = pages.get(name) ?? { name, visits: [] }; + page.visits.push({ eventId: event.id, metrics }); + pages.set(name, page); + } + return [...pages.values()].sort((a, b) => a.name.localeCompare(b.name)); +} diff --git a/packages/telemetry/src/telemetry/selectors/webVitals.ts b/packages/telemetry/src/telemetry/selectors/webVitals.ts new file mode 100644 index 000000000..91d87f690 --- /dev/null +++ b/packages/telemetry/src/telemetry/selectors/webVitals.ts @@ -0,0 +1,18 @@ +import { PERFORMANCE_SCORE_PROFILES } from "../constants/performance.ts"; +import type { SentryEvent, SentryEventWithPerformanceData } from "../types.ts"; +import { normalizePerformanceScore } from "../utils/webVitals.ts"; + +export function getWebVitals( + events: Iterable, +): SentryEventWithPerformanceData[] { + return Array.from(events) + .filter( + (event) => event.measurements && event.contexts?.trace?.op === "pageload", + ) + .map((event) => { + const copy = structuredClone(event); + normalizePerformanceScore(copy, PERFORMANCE_SCORE_PROFILES); + return copy as unknown as SentryEventWithPerformanceData; + }) + .filter((event) => event.measurements["score.total"] != null); +} diff --git a/packages/telemetry/src/telemetry/serialization.spec.ts b/packages/telemetry/src/telemetry/serialization.spec.ts new file mode 100644 index 000000000..d369d09fd --- /dev/null +++ b/packages/telemetry/src/telemetry/serialization.spec.ts @@ -0,0 +1,257 @@ +// @vitest-environment node +import { expect, test } from "vitest"; +import { SDK_CAPABILITIES } from "./capabilities.ts"; +import { serializeSnapshot, serializeTrace } from "./serialization.ts"; +import type { Span, Trace } from "./types.ts"; +function cyclicTrace(): Trace { + const parent: Span = { + span_id: "parent", + start_timestamp: 1, + timestamp: 3, + data: { + flags: new Set(["one", "two"]), + entries: new Map([["key", "value"]]), + }, + }; + const child: Span = { span_id: "child", start_timestamp: 2, timestamp: 3 }; + parent.children = [child]; + child.children = [parent]; + return { + trace_id: "trace", + transactions: [], + errors: 0, + start_timestamp: 1, + timestamp: 3, + rootTransaction: null, + rootTransactionName: "", + spans: new Map([["parent", parent]]), + spanTree: [parent], + profileGrafted: false, + }; +} +test("trace JSON retains cyclic children once with child IDs", () => { + const dto = JSON.parse(JSON.stringify(serializeTrace(cyclicTrace()))); + expect(dto.schemaVersion).toBe(1); + expect(dto.spanTree).toEqual(["parent"]); + expect(dto.spans.map(([id]: [string]) => id)).toEqual(["parent", "child"]); + expect(dto.spans[0][1]).toMatchObject({ + span_id: "parent", + childSpanIds: ["child"], + data: { flags: ["one", "two"], entries: [["key", "value"]] }, + }); + expect(dto.spans[1][1].childSpanIds).toEqual(["parent"]); + expect(dto.spans[0][1]).not.toHaveProperty("children"); +}); +test("snapshot round trips plain JSON maps and profile data", () => { + const dto = serializeSnapshot({ + revision: 4, + envelopes: [], + events: [], + traces: [cyclicTrace()], + logs: [], + sdks: [], + profiles: new Map([ + [ + "trace", + { + timestamp: 3, + active_thread_id: "main", + samples: [], + stacks: [], + frames: [], + }, + ], + ]), + }); + expect(JSON.parse(JSON.stringify(dto))).toEqual(dto); + expect(dto).toMatchObject({ + schemaVersion: 1, + revision: 4, + profiles: [["trace", { active_thread_id: "main" }]], + }); +}); +test("capabilities expose immutable supported formats", () => { + expect(SDK_CAPABILITIES).toMatchObject({ + schemaVersion: 1, + spanFormats: ["legacy", "v2"], + profileFormats: ["v1", "v2"], + }); + expect(() => + (SDK_CAPABILITIES.itemTypes as unknown as string[]).push("future"), + ).toThrow(); +}); + +test("span tree and transaction copies merge their child references without dropping descendants", () => { + const trace = cyclicTrace(); + const parent = trace.spans.get("parent")!; + const transaction = { + event_id: "transaction", + type: "transaction" as const, + timestamp: 3, + start_timestamp: 1, + contexts: { trace: { trace_id: "trace", span_id: "root" } }, + spans: [{ ...parent, children: undefined }], + }; + trace.transactions.push(transaction); + parent.transaction = transaction; + const dto = serializeTrace(trace); + expect(dto.spans).toHaveLength(2); + expect(dto.spans[0][1]).toMatchObject({ + childSpanIds: ["child"], + transactionEventId: "transaction", + }); + expect(dto.transactions).toMatchObject([ + { event_id: "transaction", spanIds: ["parent"] }, + ]); +}); + +test("valid continuous profiles and streamed plus legacy traces serialize", async () => { + const { readFileSync } = await import("node:fs"); + const { createTelemetryPipeline } = await import("../sdk/node.ts"); + const pipeline = createTelemetryPipeline(); + try { + for (const file of [ + "continuous_profiling/transaction.txt", + "continuous_profiling/profile_chunk_1.txt", + "envelope_span_v2.txt", + ]) { + const receipt = await pipeline.ingestPayload({ + body: readFileSync(new URL(`../../_fixtures/${file}`, import.meta.url)), + contentType: "application/x-sentry-envelope", + }); + expect(receipt.status).toBe("processed"); + } + const dto = serializeSnapshot(pipeline.getSnapshot()); + expect(JSON.parse(JSON.stringify(dto))).toEqual(dto); + expect(dto.traces.length).toBeGreaterThan(0); + for (const trace of dto.traces) + expect(trace.spans.length).toBeGreaterThan(0); + } finally { + pipeline.dispose(); + } +}); + +test("a wide span graph does not use call-argument expansion", () => { + const trace = cyclicTrace(); + const parent = trace.spans.get("parent")!; + parent.children = Array.from({ length: 130000 }, (_, index) => ({ + span_id: `child-${index}`, + start_timestamp: 2, + timestamp: 3, + })); + const dto = serializeTrace(trace); + expect(dto.spans).toHaveLength(130001); + expect(dto.spans[0][1].childSpanIds).toHaveLength(130000); +}); + +test("canonical spans serialize after streamed updates supersede transaction span copies", async () => { + const { createTelemetryStore } = await import("./index.ts"); + const client = createTelemetryStore(); + try { + expect( + ( + await client.ingestEnvelope([ + {}, + [ + [ + { type: "transaction" }, + { + event_id: "old", + type: "transaction", + start_timestamp: 1, + timestamp: 2, + contexts: { trace: { trace_id: "mixed", span_id: "root" } }, + spans: [ + { + span_id: "child", + parent_span_id: "root", + start_timestamp: 1.1, + timestamp: 1.2, + description: "old", + }, + ], + }, + ], + ], + ]) + ).status, + ).toBe("processed"); + expect( + ( + await client.ingestEnvelope([ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "mixed", + span_id: "child", + parent_span_id: "root", + start_timestamp: 1.1, + end_timestamp: 1.3, + name: "updated", + }, + ], + }, + ], + ], + ]) + ).status, + ).toBe("processed"); + const dto = serializeTrace(client.getTrace("mixed")!); + expect(dto.spans.find(([id]) => id === "child")?.[1]).toMatchObject({ + description: "updated", + timestamp: 1300, + }); + } finally { + client.dispose(); + } +}); + +test("summary JSON converts profile trace ID sets to arrays", async () => { + const { getProfileFunctionSummaries } = + await import("./selectors/profiles.ts"); + const { serializeSummaries } = await import("./serialization.ts"); + const summaries = getProfileFunctionSummaries( + new Map([ + [ + "trace-one", + { + samples: [ + { start_timestamp: 0, stack_id: 0, thread_id: "main" }, + { start_timestamp: 10, stack_id: 0, thread_id: "main" }, + ], + stacks: [[0]], + frames: [{ function: "work" }], + }, + ], + ]), + ); + const dto = serializeSummaries(summaries); + expect(dto).toEqual({ + schemaVersion: 1, + summaries: [ + { + name: "@work", + totalDurationMs: 10, + sampleCount: 1, + traceIds: ["trace-one"], + }, + ], + }); + expect(JSON.parse(JSON.stringify(dto))).toEqual(dto); + expect(() => + serializeSummaries([ + { + description: "query", + count: 1, + totalDurationMs: 1, + averageDurationMs: 1, + spans: [], + }, + ] as never), + ).toThrow(TypeError); +}); diff --git a/packages/telemetry/src/telemetry/serialization.ts b/packages/telemetry/src/telemetry/serialization.ts new file mode 100644 index 000000000..7f43484de --- /dev/null +++ b/packages/telemetry/src/telemetry/serialization.ts @@ -0,0 +1,218 @@ +import type { + DeepReadonly, + ProcessedSpan, + ProcessedTrace, + ProfileFunctionSummary, + TelemetrySnapshot, +} from "./public.ts"; +import type { CompactQuerySummary, CompactResourceSummary } from "./reader.ts"; + +export type JsonValue = + | string + | number + | boolean + | null + | JsonValue[] + | { [key: string]: JsonValue }; +export type SerializedSpan = { + [key: string]: JsonValue; + span_id: string; + childSpanIds: string[]; +}; +export type SerializedTrace = { + [key: string]: JsonValue; + schemaVersion: 1; + trace_id: string; + spans: [string, SerializedSpan][]; + spanTree: string[]; +}; +export type SerializedSnapshot = { + schemaVersion: 1; + revision: number; + envelopes: JsonValue[]; + events: JsonValue[]; + traces: SerializedTrace[]; + logs: JsonValue[]; + sdks: JsonValue[]; + profiles: [string, JsonValue][]; +}; +export type SerializableSummary = + | ((CompactQuerySummary | CompactResourceSummary) & { + readonly spans?: never; + }) + | ProfileFunctionSummary; +export type SerializedSummaries = { schemaVersion: 1; summaries: JsonValue[] }; + +/** Explicitly reject unsupported data rather than quietly replacing graph cycles with null. */ +function plain(value: unknown, ancestors = new Set()): JsonValue { + if (value === null) return null; + if (typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new TypeError("Telemetry JSON requires finite numbers"); + return value; + } + if (typeof value !== "object") + throw new TypeError("Unsupported telemetry JSON value"); + if (ancestors.has(value)) + throw new TypeError("Cycle outside span children cannot be serialized"); + ancestors.add(value); + try { + if (value instanceof Map) + return Array.from(value, ([key, item]) => [ + plain(key, ancestors), + plain(item, ancestors), + ]); + if (value instanceof Set) + return Array.from(value, (item) => plain(item, ancestors)); + if (value instanceof Uint8Array) return Array.from(value); + if (Array.isArray(value)) + return value.map((item) => + item === undefined ? null : plain(item, ancestors), + ); + const record = value as Record; + const result: { [key: string]: JsonValue } = {}; + const isSpan = + typeof record.span_id === "string" && + typeof record.start_timestamp === "number" && + typeof record.timestamp === "number"; + for (const [key, item] of Object.entries(record).sort(([a], [b]) => + a.localeCompare(b), + )) { + if ( + item === undefined || + (isSpan && (key === "children" || key === "transaction")) + ) + continue; + Object.defineProperty(result, key, { + value: plain(item, ancestors), + enumerable: true, + writable: true, + configurable: true, + }); + } + if (isSpan) { + result.childSpanIds = ( + (record.children ?? []) as readonly ProcessedSpan[] + ).map((child) => child.span_id); + if (record.transaction) + result.transactionEventId = ( + record.transaction as { event_id: string } + ).event_id; + } + return result; + } finally { + ancestors.delete(value); + } +} + +/** Flat span entries preserve child cycles and retain children missing from the trace's span map. */ +export function serializeTrace(trace: ProcessedTrace): SerializedTrace { + const queue: ProcessedSpan[] = [...trace.spans.values(), ...trace.spanTree]; + const canonicalIds = new Set( + Array.from(trace.spans.values(), (span) => span.span_id), + ); + const transactionsSeen = new Set>(); + function enqueueTransaction( + transaction: NonNullable, + ) { + if (transactionsSeen.has(transaction)) return; + transactionsSeen.add(transaction); + for (const span of transaction.spans ?? []) queue.push(span); + } + for (const transaction of trace.transactions) enqueueTransaction(transaction); + if (trace.rootTransaction) enqueueTransaction(trace.rootTransaction); + const seen = new Set(); + const byId = new Map(); + for (let index = 0; index < queue.length; index++) { + const span = queue[index]; + if (seen.has(span)) continue; + seen.add(span); + const serialized = plain(span) as SerializedSpan; + const existing = byId.get(span.span_id); + // The trace map is authoritative over historical copies in transactions. + // Children found only in those copies still belong in the flat graph. + if (existing) { + if (!canonicalIds.has(span.span_id)) + for (const [key, value] of Object.entries(serialized)) { + if (key === "childSpanIds") continue; + if ( + Object.hasOwn(existing, key) && + JSON.stringify(existing[key]) !== JSON.stringify(value) + ) + throw new TypeError(`Conflicting span ID: ${span.span_id}`); + Object.defineProperty(existing, key, { + value, + enumerable: true, + writable: true, + configurable: true, + }); + } + existing.childSpanIds = Array.from( + new Set([...existing.childSpanIds, ...serialized.childSpanIds]), + ); + } else byId.set(span.span_id, serialized); + for (const child of span.children ?? []) queue.push(child); + if (span.transaction) enqueueTransaction(span.transaction); + } + const { + spans: _spans, + spanTree: _tree, + transactions, + rootTransaction, + ...metadata + } = trace; + const transactionJson = ( + transaction: DeepReadonly>, + ): JsonValue => { + const { spans, ...rest } = transaction; + return { + ...(plain(rest) as { [key: string]: JsonValue }), + spanIds: (spans ?? []).map((span) => span.span_id), + }; + }; + return { + ...(plain(metadata) as { [key: string]: JsonValue }), + schemaVersion: 1, + trace_id: trace.trace_id, + transactions: transactions.map(transactionJson), + rootTransaction: rootTransaction ? transactionJson(rootTransaction) : null, + spans: Array.from(byId), + spanTree: trace.spanTree.map((span) => span.span_id), + }; +} + +/** A JSON-ready DTO: Map entries and Set values become arrays; no custom JSON replacer is needed. */ +export function serializeSnapshot( + snapshot: TelemetrySnapshot, +): SerializedSnapshot { + return { + schemaVersion: 1, + revision: snapshot.revision, + envelopes: snapshot.envelopes.map((envelope) => plain(envelope)), + events: snapshot.events.map((event) => plain(event)), + traces: snapshot.traces.map(serializeTrace), + logs: snapshot.logs.map((log) => plain(log)), + sdks: snapshot.sdks.map((sdk) => plain(sdk)), + profiles: Array.from(snapshot.profiles, ([id, profile]) => [ + id, + plain(profile), + ]), + }; +} + +/** Serialize compact read results, including profile trace ID sets, without losing a detailed span graph. */ +export function serializeSummaries( + summaries: readonly SerializableSummary[], +): SerializedSummaries { + return { + schemaVersion: 1, + summaries: summaries.map((summary) => { + if ("spans" in summary) + throw new TypeError( + "Use compact summaries or serializeTrace for detailed span graphs", + ); + return plain(summary); + }), + }; +} diff --git a/packages/telemetry/src/telemetry/store/helpers.ts b/packages/telemetry/src/telemetry/store/helpers.ts new file mode 100644 index 000000000..d79423301 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/helpers.ts @@ -0,0 +1,8 @@ +import type { Span } from "../types.ts"; + +export function getAllSpansInTree(root: Span): Span[] { + const spans = [root]; + return root.children + ? spans.concat(root.children.flatMap(getAllSpansInTree)) + : spans; +} diff --git a/packages/telemetry/src/telemetry/store/slices/envelopesSlice.ts b/packages/telemetry/src/telemetry/store/slices/envelopesSlice.ts new file mode 100644 index 000000000..32a258965 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/slices/envelopesSlice.ts @@ -0,0 +1,205 @@ +import { stringifyValue } from "../../utils/stringifyValue.ts"; +import type { Envelope } from "@sentry/core"; +import type { StateCreator } from "zustand/vanilla"; +import { + ATTACHMENT_EVENT_TYPES, + RAW_TYPES, + SUPPORTED_EVENT_TYPES, +} from "../../constants/sentry.ts"; +import type { ProcessingRuntime } from "../../internal/types.ts"; +import type { EventAttachment, Sdk, SentryEvent } from "../../types.ts"; +import { sdkToPlatform } from "../../utils/sdkToPlatform.ts"; +import { generateUuidv4 } from "../../uuid.ts"; +import type { + EnvelopesSliceActions, + EnvelopesSliceState, + SentryStore, +} from "../types.ts"; +import { normalizeSpanItem } from "../utils/spanProcessor.ts"; + +export const createEnvelopesSlice = + ( + runtime: ProcessingRuntime, + ): StateCreator< + SentryStore, + [], + [], + EnvelopesSliceState & EnvelopesSliceActions + > => + (set, get) => { + const processing = new Map[]>(); + const generatedIds = new WeakMap(); + function process(envelope: Envelope) { + const generation = runtime.generation; + const pending: Promise[] = []; + const envelopeId = stringifyValue( + envelope[0].__spotlight_envelope_id ?? + generatedIds.get(envelope) ?? + generateUuidv4(), + ); + generatedIds.set(envelope, envelopeId); + if (get().envelopes.has(envelopeId)) + return { + count: get().envelopes.size, + pending: processing.get(envelopeId) ?? [], + }; + processing.set(envelopeId, pending); + function finish(count: number) { + void Promise.allSettled(pending).then(() => { + if (processing.get(envelopeId) === pending) + processing.delete(envelopeId); + }); + return { count, pending }; + } + const [header, items] = envelope; + const sentAt = new Date(header.sent_at as string).getTime(); + const lastSeen = Number.isFinite(sentAt) ? sentAt : runtime.receivedAtMs; + // Read the inferred source type from the envelope header (set by server) + const sourceType = ( + header as { + __spotlight_inferred_source?: "browser" | "server" | "mobile"; + } + ).__spotlight_inferred_source; + let sdk: Sdk; + + if (header.sdk?.name && header.sdk.version) { + sdk = { + name: header.sdk.name, + version: header.sdk.version, + lastSeen: lastSeen, + }; + } else if (items.length > 0) { + sdk = get().inferSdkFromEvent(items[0][1] as SentryEvent); + sdk.lastSeen = lastSeen; + } else { + sdk = { + name: "unknown", + version: "0.0.0", + lastSeen, + }; + } + + get().storeSdkRecord(sdk); + if (runtime.generation !== generation) + return finish(get().envelopes.size); + + const traceContext = header.trace; + + const attachmentsByEventId = new Map(); + const envelopeScopedAttachments: EventAttachment[] = []; + let eventCount = 0; + + for (const [itemHeader, itemData] of items) { + if (SUPPORTED_EVENT_TYPES.has(itemHeader.type)) { + eventCount += 1; + } + + if (!ATTACHMENT_EVENT_TYPES.has(itemHeader.type)) { + continue; + } + + const rawData = (itemData as { data?: unknown })?.data; + if (typeof rawData !== "string") { + continue; + } + + const attachment: EventAttachment = { + header: itemHeader, + data: rawData, + }; + + const eventId = + "event_id" in itemHeader + ? (itemHeader.event_id as string | undefined) + : undefined; + + if (eventId) { + const existing = attachmentsByEventId.get(eventId) ?? []; + existing.push(attachment); + attachmentsByEventId.set(eventId, existing); + } else { + envelopeScopedAttachments.push(attachment); + } + } + + for (const [itemIndex, [itemHeader, itemData]] of items.entries()) { + if (runtime.generation !== generation) break; + if (itemHeader.type === "span") { + for (const event of normalizeSpanItem(itemData, sdk)) { + if (runtime.generation !== generation) break; + event.platform = sdkToPlatform(sdk.name); + if (sourceType) event.__sourceType = sourceType; + pending.push(get().pushEvent(event)); + } + continue; + } + if (SUPPORTED_EVENT_TYPES.has(itemHeader.type)) { + const item = structuredClone(itemData) as SentryEvent; + item.type ??= itemHeader.type as SentryEvent["type"]; + if (!RAW_TYPES.has(itemHeader.type)) { + item.platform = sdkToPlatform(sdk.name); + } + if (traceContext) { + if (!item.contexts) { + item.contexts = {}; + } + item.contexts.trace ??= traceContext; + } + // Attach the inferred source type to the event for UI display + if (sourceType) { + (item as { __sourceType?: typeof sourceType }).__sourceType = + sourceType; + } + const eventId = + item.event_id ?? + ("event_id" in itemHeader + ? (itemHeader.event_id as string | undefined) + : undefined); + item.event_id = eventId ?? `${envelopeId}:${itemIndex}`; + let attachmentsForEvent = eventId + ? attachmentsByEventId.get(eventId) + : undefined; + + if ( + !attachmentsForEvent?.length && + eventCount === 1 && + envelopeScopedAttachments.length > 0 + ) { + attachmentsForEvent = envelopeScopedAttachments; + } + + if (attachmentsForEvent?.length) { + item.attachments = attachmentsForEvent; + } + pending.push(get().pushEvent(item)); + } + } + + const { envelopes } = get(); + if (runtime.generation !== generation) return finish(envelopes.size); + const newEnvelopes = new Map(envelopes); + newEnvelopes.set(envelopeId, envelope); + set({ envelopes: newEnvelopes }); + return finish(newEnvelopes.size); + } + return { + envelopes: new Map(), + // Compatibility path used by the UI: synchronous processing except optional enrichment. + pushEnvelope: (envelope) => { + const { count, pending } = process(envelope); + for (const task of pending) void task.catch(() => {}); + return count; + }, + ingestEnvelope: async (envelope) => { + const { count, pending } = process(envelope); + const results = await Promise.allSettled(pending); + const failure = results.find((result) => result.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; + return count; + }, + getEnvelopeById: (id: string) => { + return get().envelopes.get(id); + }, + getEnvelopes: () => Array.from(get().envelopes.values()), + }; + }; diff --git a/packages/telemetry/src/telemetry/store/slices/eventsSlice.ts b/packages/telemetry/src/telemetry/store/slices/eventsSlice.ts new file mode 100644 index 000000000..acf9069db --- /dev/null +++ b/packages/telemetry/src/telemetry/store/slices/eventsSlice.ts @@ -0,0 +1,330 @@ +import type { StateCreator } from "zustand/vanilla"; +import { graftProfileSpans } from "../../data/profiles.ts"; +import type { EngineOptions, ProcessingRuntime } from "../../internal/types.ts"; +import type { SentryEvent, SentryLogEventItem } from "../../types.ts"; +import type { SentryTransactionEvent } from "../../types.ts"; +import { + isErrorEvent, + isLogEvent, + isProfileChunkEvent, + isProfileEvent, + isTraceEvent, +} from "../../utils/sentry.ts"; +import { generateUuidv4 } from "../../uuid.ts"; +import type { + EventsSliceActions, + EventsSliceState, + SentryProfileWithTraceMeta, + SentryStore, +} from "../types.ts"; +import { toTimestamp } from "../utils.ts"; +import { processLogItems } from "../utils/logProcessor.ts"; +import { + mergeChunksToProfile, + processProfileChunkEvent, +} from "../utils/profileChunkProcessor.ts"; +import { processProfileEvent } from "../utils/profileProcessor.ts"; +import { processSpanEvent } from "../utils/spanProcessor.ts"; +import { initializeTrace } from "../utils/traceInitializer.ts"; +import { + processTransactionEvent, + updateTraceMetadata, +} from "../utils/traceProcessor.ts"; + +export const createEventsSlice = + ( + options: EngineOptions, + runtime: ProcessingRuntime, + ): StateCreator => + (set, get) => { + const pendingIds = new Map(); + return { + eventsById: new Map(), + + pushEvent: async (input: SentryEvent & { event_id?: string }) => { + const event = structuredClone(input); + const generation = runtime.generation; + if (!event.event_id) { + event.event_id = generateUuidv4(); + } + + const { eventsById: _eventsById } = get(); + if ( + _eventsById.has(event.event_id) || + pendingIds.get(event.event_id) === generation + ) + return; + pendingIds.set(event.event_id, generation); + try { + if (isErrorEvent(event)) { + for (const exception of event.exception?.values ?? []) + exception.stacktrace?.frames?.reverse(); + try { + if (options.enrichStacktrace) + await options.enrichStacktrace(event); + } catch { + // Source context is optional; a failed adapter must not drop captured telemetry. + } + if ( + runtime.generation !== generation || + get().eventsById.has(event.event_id) + ) + return; + } + + event.timestamp = toTimestamp( + event.timestamp ?? runtime.receivedAtMs / 1000, + ); + if (event.start_timestamp) { + event.start_timestamp = toTimestamp(event.start_timestamp); + } + + if (event.type === "span") { + event.span.start_timestamp = toTimestamp( + event.span.start_timestamp, + ); + event.span.timestamp = toTimestamp(event.span.timestamp); + } else if (isTraceEvent(event)) { + event.spans = event.spans?.map((span) => ({ + ...span, + start_timestamp: toTimestamp(span.start_timestamp), + timestamp: toTimestamp(span.timestamp), + })); + } + + if (isLogEvent(event) && event.items?.length) { + const { logsById, logsByTraceId, tracesById } = get(); + const existingLogIds = new Set(logsById.keys()); + const { processedLogs } = processLogItems(event, existingLogIds); + + // Update store with processed logs + // Build up the Maps incrementally to avoid stale state capture + const newLogsById = new Map(logsById); + const newLogsByTraceId = new Map(logsByTraceId); + const newTracesById = new Map(tracesById); + + for (const logItem of processedLogs) { + newLogsById.set(logItem.id, logItem); + + if ( + typeof logItem.trace_id === "string" && + logItem.trace_id.trim() + ) { + const logSet = + newLogsByTraceId.get(logItem.trace_id) || + new Set(); + logSet.add(logItem); + newLogsByTraceId.set(logItem.trace_id, logSet); + // A log batch carries trace IDs on its individual records, not + // necessarily on the containing event. Keep log-only traces + // discoverable until transactions or spans arrive later. + if (Number.isFinite(logItem.timestamp)) { + const trace = + newTracesById.get(logItem.trace_id) ?? + initializeTrace({ + ...event, + timestamp: logItem.timestamp, + start_timestamp: undefined, + contexts: { trace: { trace_id: logItem.trace_id } }, + }); + trace.start_timestamp = Math.min( + trace.start_timestamp, + logItem.timestamp, + ); + trace.timestamp = Math.max( + trace.timestamp, + logItem.timestamp, + ); + newTracesById.set(logItem.trace_id, trace); + } + } + } + + // Set state once with all accumulated changes + set({ + logsById: newLogsById, + logsByTraceId: newLogsByTraceId, + tracesById: newTracesById, + }); + if (runtime.generation !== generation) return; + } + + const { eventsById } = get(); + const newEventIds = new Map(eventsById); + newEventIds.set(event.event_id, event); + set({ eventsById: newEventIds }); + if (runtime.generation !== generation) return; + + // Notify event subscribers + for (const [type, callback] of get().subscribers.values()) { + if (type === "event") { + (callback as (event: SentryEvent) => void)(event); + if (runtime.generation !== generation) return; + } + } + + const traceCtx = event.contexts?.trace; + if (traceCtx?.trace_id) { + const { tracesById, profilesByTraceId, profileChunksByProfilerId } = + get(); + const existingTrace = tracesById.get(traceCtx.trace_id); + + // Initialize or get existing trace + let trace = existingTrace ?? initializeTrace(event); + + // Update timestamps for all events + trace.start_timestamp = Math.min( + event.start_timestamp ?? event.timestamp, + trace.start_timestamp, + ); + trace.timestamp = Math.max(event.timestamp, trace.timestamp); + + let mergedV2Profile: SentryProfileWithTraceMeta | undefined; + + if (isTraceEvent(event)) { + // Process transaction event and update trace + const result = processTransactionEvent(event, { + existingTrace: trace, + profilesByTraceId, + profileChunksByProfilerId, + }); + trace = result.trace; + mergedV2Profile = result.mergedProfile; + } else if (event.type === "span") { + const result = processSpanEvent(event, { + existingTrace: trace, + profilesByTraceId, + profileChunksByProfilerId, + }); + trace = result.trace; + mergedV2Profile = result.mergedProfile; + } else if (isErrorEvent(event)) { + // For error events, increment error count + trace.errors += 1; + } + + // Update trace metadata and status for all events with trace context + updateTraceMetadata(trace); + + if (traceCtx.status !== "ok") { + trace.status = traceCtx.status; + } + + // Always save trace to store (whether new or updated) + const newTracesById = new Map(tracesById); + newTracesById.set(trace.trace_id, trace); + + // If we merged a V2 profile, store it by trace_id so the UI can find it + if (mergedV2Profile) { + const newProfilesByTraceId = new Map(profilesByTraceId); + newProfilesByTraceId.set(trace.trace_id, mergedV2Profile); + set({ + tracesById: newTracesById, + profilesByTraceId: newProfilesByTraceId, + }); + } else { + set({ tracesById: newTracesById }); + } + } + + if (runtime.generation !== generation) return; + if (isProfileEvent(event)) { + const { profilesByTraceId, tracesById } = get(); + const { profiles } = processProfileEvent(event, { tracesById }); + + // Update store with processed profiles + const newProfilesByTraceId = new Map(profilesByTraceId); + for (const { traceId, profile } of profiles) { + newProfilesByTraceId.set(traceId, profile); + } + set({ profilesByTraceId: newProfilesByTraceId }); + } + + // Handle V2 profile chunks (continuous profiling) + if (isProfileChunkEvent(event)) { + const { + profileChunksByProfilerId, + tracesById, + eventsById, + profilesByTraceId, + } = get(); + const { chunk } = processProfileChunkEvent(event); + + // Update store with processed profile chunk + const newProfileChunksByProfilerId = new Map( + profileChunksByProfilerId, + ); + const existingChunks = + newProfileChunksByProfilerId.get(chunk.profiler_id) || []; + + // Check if this chunk already exists (by chunk_id) + if (!existingChunks.some((c) => c.chunk_id === chunk.chunk_id)) { + const updatedChunks = [...existingChunks, chunk]; + newProfileChunksByProfilerId.set( + chunk.profiler_id, + updatedChunks, + ); + + // Check if there's an existing transaction with this profiler_id that needs re-grafting + // This handles the case where profile chunks arrive AFTER the transaction + let traceToUpdate: string | undefined; + let activeThreadId: string | undefined; + for (const evt of eventsById.values()) { + if (isTraceEvent(evt)) { + const txn = evt as SentryTransactionEvent; + const profileCtx = txn.contexts?.profile as + | { profiler_id?: string } + | undefined; + if (profileCtx?.profiler_id === chunk.profiler_id) { + traceToUpdate = txn.contexts.trace.trace_id; + activeThreadId = txn.contexts.trace.data?.["thread.id"] as + | string + | undefined; + break; + } + } + } + + if (traceToUpdate) { + // Re-graft with updated chunks + const existingTrace = tracesById.get(traceToUpdate); + if (existingTrace && !existingTrace.profileGrafted) { + const mergedProfile = mergeChunksToProfile( + updatedChunks, + activeThreadId, + ); + if (mergedProfile) { + // Create a shallow copy of the trace to trigger React re-render + const trace = { + ...existingTrace, + spans: new Map(existingTrace.spans), + spanTree: [...existingTrace.spanTree], + }; + graftProfileSpans(trace, mergedProfile); + + // Store updated trace and profile + const newTracesById = new Map(tracesById); + newTracesById.set(traceToUpdate, trace); + const newProfilesByTraceId = new Map(profilesByTraceId); + newProfilesByTraceId.set(traceToUpdate, mergedProfile); + set({ + profileChunksByProfilerId: newProfileChunksByProfilerId, + tracesById: newTracesById, + profilesByTraceId: newProfilesByTraceId, + }); + return; + } + } + } + + set({ profileChunksByProfilerId: newProfileChunksByProfilerId }); + } + } + } finally { + if (pendingIds.get(event.event_id) === generation) + pendingIds.delete(event.event_id); + } + }, + getEvents: () => Array.from(get().eventsById.values()), + }; + }; diff --git a/packages/telemetry/src/telemetry/store/slices/logsSlice.ts b/packages/telemetry/src/telemetry/store/slices/logsSlice.ts new file mode 100644 index 000000000..32c960b6a --- /dev/null +++ b/packages/telemetry/src/telemetry/store/slices/logsSlice.ts @@ -0,0 +1,23 @@ +import type { StateCreator } from "zustand/vanilla"; +import type { + LogsSliceActions, + LogsSliceState, + SentryStore, +} from "../types.ts"; + +export const createLogsSlice: StateCreator< + SentryStore, + [], + [], + LogsSliceState & LogsSliceActions +> = (_set, get) => ({ + logsById: new Map(), + logsByTraceId: new Map(), + + getLogById: (id: string) => get().logsById.get(id), + getLogs: () => Array.from(get().logsById.values()), + getLogsByTraceId: (traceId: string) => { + const logsByTraceId = get().logsByTraceId.get(traceId); + return logsByTraceId ? Array.from(logsByTraceId) : []; + }, +}); diff --git a/packages/telemetry/src/telemetry/store/slices/profilesSlice.ts b/packages/telemetry/src/telemetry/store/slices/profilesSlice.ts new file mode 100644 index 000000000..9536388ac --- /dev/null +++ b/packages/telemetry/src/telemetry/store/slices/profilesSlice.ts @@ -0,0 +1,29 @@ +import type { StateCreator } from "zustand/vanilla"; +import { getProfileFunctionSummaries } from "../../selectors/profiles.ts"; +import type { + ProfilesSliceActions, + ProfilesSliceState, + SentryStore, +} from "../types.ts"; + +export const createProfilesSlice: StateCreator< + SentryStore, + [], + [], + ProfilesSliceState & ProfilesSliceActions +> = (_set, get) => ({ + profilesByTraceId: new Map(), + profileChunksByProfilerId: new Map(), + + // Unified profile access - works for both V1 and merged V2 profiles + getProfileByTraceId: (id: string) => get().profilesByTraceId.get(id), + + // Keep legacy field names private to the store adapter. + getAggregateCallData: () => + getProfileFunctionSummaries(get().profilesByTraceId).map((summary) => ({ + name: summary.name, + totalTime: summary.totalDurationMs, + samples: summary.sampleCount, + traceIds: summary.traceIds, + })), +}); diff --git a/packages/telemetry/src/telemetry/store/slices/sdksSlice.ts b/packages/telemetry/src/telemetry/store/slices/sdksSlice.ts new file mode 100644 index 000000000..4881d8464 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/slices/sdksSlice.ts @@ -0,0 +1,48 @@ +import type { StateCreator } from "zustand/vanilla"; +import type { Sdk, SentryEvent } from "../../types.ts"; +import type { + SDKsSliceActions, + SDKsSliceState, + SentryStore, +} from "../types.ts"; + +const getSDKKey = (sdk: Sdk): string => `${sdk.name}@${sdk.version}`; + +export const createSDKsSlice: StateCreator< + SentryStore, + [], + [], + SDKsSliceState & SDKsSliceActions +> = (set, get) => ({ + sdks: new Map(), + + inferSdkFromEvent: (event: SentryEvent) => { + const sdk: Sdk = { + name: "unknown", + version: "unknown", + lastSeen: new Date().getTime(), + }; + + if (event.sdk) { + sdk.name = event.sdk.name || sdk.name; + sdk.version = event.sdk.version || sdk.version; + } else if (event.platform) { + sdk.name = event.platform; + } + + return sdk; + }, + storeSdkRecord: (sdk: Sdk) => { + const sdks = get().sdks; + const existingSdk = sdks.get(getSDKKey(sdk)); + if (existingSdk) { + existingSdk.lastSeen = sdk.lastSeen; + } else { + const newSdks = new Map(sdks); + newSdks.set(getSDKKey(sdk), sdk); + set({ sdks: newSdks }); + } + return existingSdk || sdk; + }, + getSdks: () => Array.from(get().sdks.values()), +}); diff --git a/packages/telemetry/src/telemetry/store/slices/sharedSlice.ts b/packages/telemetry/src/telemetry/store/slices/sharedSlice.ts new file mode 100644 index 000000000..9c5d2a9a5 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/slices/sharedSlice.ts @@ -0,0 +1,35 @@ +import type { StateCreator } from "zustand/vanilla"; +import type { ProcessingRuntime } from "../../internal/types.ts"; +import type { SentryStore, SharedSliceActions } from "../types.ts"; + +export const createSharedSlice = + ( + runtime: ProcessingRuntime, + ): StateCreator => + (set, get) => ({ + getEventById: (id: string) => get().eventsById.get(id), + getTraceById: (id: string) => get().tracesById.get(id), + getEventsByTrace: (traceId: string, spanId?: string | null) => + get() + .getEvents() + .filter((event) => { + const trace = event.contexts?.trace; + return ( + trace?.trace_id === traceId && + (spanId === undefined || trace.span_id === spanId) + ); + }), + resetData: () => { + runtime.generation += 1; + set({ + envelopes: new Map(), + eventsById: new Map(), + tracesById: new Map(), + sdks: new Map(), + profilesByTraceId: new Map(), + profileChunksByProfilerId: new Map(), + logsById: new Map(), + logsByTraceId: new Map(), + }); + }, + }); diff --git a/packages/telemetry/src/telemetry/store/slices/subscriptionsSlice.ts b/packages/telemetry/src/telemetry/store/slices/subscriptionsSlice.ts new file mode 100644 index 000000000..9129cf4e7 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/slices/subscriptionsSlice.ts @@ -0,0 +1,32 @@ +import type { StateCreator } from "zustand/vanilla"; +import { generateUuidv4 } from "../../uuid.ts"; +import type { + SentryStore, + Subscription, + SubscriptionsSliceActions, + SubscriptionsSliceState, +} from "../types.ts"; + +export const createSubscriptionsSlice: StateCreator< + SentryStore, + [], + [], + SubscriptionsSliceState & SubscriptionsSliceActions +> = (set, get) => ({ + subscribers: new Map(), + + subscribe: (...args: Subscription) => { + const id = generateUuidv4(); + const { subscribers } = get(); + const newSubscribers = new Map(subscribers); + newSubscribers.set(id, args); + set({ subscribers: newSubscribers }); + + return () => { + const { subscribers: currentSubscribers } = get(); + const updatedSubscribers = new Map(currentSubscribers); + updatedSubscribers.delete(id); + set({ subscribers: updatedSubscribers }); + }; + }, +}); diff --git a/packages/telemetry/src/telemetry/store/slices/tracesSlice.ts b/packages/telemetry/src/telemetry/store/slices/tracesSlice.ts new file mode 100644 index 000000000..3ddef8fcb --- /dev/null +++ b/packages/telemetry/src/telemetry/store/slices/tracesSlice.ts @@ -0,0 +1,17 @@ +import type { StateCreator } from "zustand/vanilla"; +import type { + SentryStore, + TracesSliceActions, + TracesSliceState, +} from "../types.ts"; + +export const createTracesSlice: StateCreator< + SentryStore, + [], + [], + TracesSliceState & TracesSliceActions +> = (_, get) => ({ + tracesById: new Map(), + + getTraces: () => Array.from(get().tracesById.values()), +}); diff --git a/packages/telemetry/src/telemetry/store/types.ts b/packages/telemetry/src/telemetry/store/types.ts new file mode 100644 index 000000000..38ecc9815 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/types.ts @@ -0,0 +1,116 @@ +import type { Envelope } from "@sentry/core"; +import type { + AggregateCallData, + Sdk, + SentryEvent, + SentryLogEventItem, + Trace, +} from "../types.ts"; +import type { ProcessedProfileChunk } from "./utils/profileChunkProcessor.ts"; + +import type { SentryProfileWithTraceMeta } from "../types.ts"; +export type { SentryProfileWithTraceMeta } from "../types.ts"; + +export type OnlineSubscription = ["online", (status: boolean) => void]; +export type EventSubscription = ["event", (event: SentryEvent) => void]; +export type TraceSubscription = ["trace", (trace: Trace) => void]; +export type Subscription = + | OnlineSubscription + | EventSubscription + | TraceSubscription; + +export interface EventsSliceState { + eventsById: Map; +} + +export interface EventsSliceActions { + pushEvent: (event: SentryEvent & { event_id?: string }) => Promise; + getEvents: () => SentryEvent[]; +} + +export interface TracesSliceState { + tracesById: Map; +} + +export interface TracesSliceActions { + getTraces: () => Trace[]; +} + +export interface ProfilesSliceState { + /** Final merged profiles keyed by trace_id - works for both V1 and V2 */ + profilesByTraceId: Map; + /** Internal: V2 chunks being accumulated, keyed by profiler_id */ + profileChunksByProfilerId: Map; +} + +export interface ProfilesSliceActions { + /** Get profile for a trace - unified access for both V1 and V2 profiles */ + getProfileByTraceId: (id: string) => SentryProfileWithTraceMeta | undefined; + getAggregateCallData: () => AggregateCallData[]; +} + +export interface SubscriptionsSliceState { + subscribers: Map; +} + +export interface SubscriptionsSliceActions { + subscribe: (...args: Subscription) => () => void; +} + +export interface EnvelopesSliceState { + envelopes: Map; +} + +export interface EnvelopesSliceActions { + pushEnvelope: (params: Envelope) => number; + ingestEnvelope: (params: Envelope) => Promise; + getEnvelopeById: (id: string) => Envelope | undefined; + getEnvelopes: () => Array; +} + +export interface SDKsSliceState { + sdks: Map; +} + +export interface LogsSliceState { + logsById: Map; + logsByTraceId: Map>; +} + +export interface LogsSliceActions { + getLogById: (id: string) => SentryLogEventItem | undefined; + getLogs: () => SentryLogEventItem[]; + getLogsByTraceId: (traceId: string) => SentryLogEventItem[]; +} + +export interface SDKsSliceActions { + inferSdkFromEvent: (event: SentryEvent) => Sdk; + storeSdkRecord: (sdk: Sdk) => Sdk; + getSdks: () => Sdk[]; +} + +export interface SharedSliceActions { + getEventById: (id: string) => SentryEvent | undefined; + getTraceById: (id: string) => Trace | undefined; + getEventsByTrace: (traceId: string, spanId?: string | null) => SentryEvent[]; + resetData: () => void; +} + +export type SentryStoreState = EventsSliceState & + TracesSliceState & + ProfilesSliceState & + SubscriptionsSliceState & + EnvelopesSliceState & + LogsSliceState & + SDKsSliceState; + +export type SentryStoreActions = EventsSliceActions & + TracesSliceActions & + ProfilesSliceActions & + SubscriptionsSliceActions & + EnvelopesSliceActions & + LogsSliceActions & + SDKsSliceActions & + SharedSliceActions; + +export type SentryStore = SentryStoreState & SentryStoreActions; diff --git a/packages/telemetry/src/telemetry/store/utils.ts b/packages/telemetry/src/telemetry/store/utils.ts new file mode 100644 index 000000000..2b12d9e9b --- /dev/null +++ b/packages/telemetry/src/telemetry/store/utils.ts @@ -0,0 +1,11 @@ +export function toTimestamp(date: string | number) { + if (typeof date === "string") return new Date(date).getTime(); + return date * 1000; +} + +export function relativeNsToTimestamp(startTs: number, ns: number | string) { + const nsStr = ns.toString(); + return nsStr.length > 3 + ? startTs + Number.parseInt(nsStr.slice(0, -3), 10) / 1000 + : startTs; +} diff --git a/packages/telemetry/src/telemetry/store/utils/logProcessor.ts b/packages/telemetry/src/telemetry/store/utils/logProcessor.ts new file mode 100644 index 000000000..14fc77adb --- /dev/null +++ b/packages/telemetry/src/telemetry/store/utils/logProcessor.ts @@ -0,0 +1,48 @@ +import type { SentryLogEvent, SentryLogEventItem } from "../../types.ts"; +import { toTimestamp } from "../utils.ts"; + +export interface LogProcessingResult { + processedLogs: SentryLogEventItem[]; +} + +/** + * Processes log items from a log event, normalizing and enriching them with metadata. + * @param event The log event to process + * @param existingLogIds Set of already processed log IDs to avoid duplicates + * @returns Processed log items ready to be stored + */ +export function processLogItems( + event: SentryLogEvent, + existingLogIds: Set, +): LogProcessingResult { + const processedLogs: SentryLogEventItem[] = []; + + if (!event.items?.length) { + return { processedLogs }; + } + + for (const [index, logItem] of event.items.entries()) { + const logId = logItem.id || `${event.event_id}:log:${index}`; + + // Skip if already processed + if (existingLogIds.has(logId)) { + continue; + } + + // Normalize severity number + if (logItem.severity_number == null) { + logItem.severity_number = 0; + } + + // Extract SDK name from attributes + logItem.sdk = logItem.attributes?.["sentry.sdk.name"]?.value as string; + + // Normalize timestamp + logItem.timestamp = toTimestamp(logItem.timestamp); + logItem.id = logId; + + processedLogs.push(logItem); + } + + return { processedLogs }; +} diff --git a/packages/telemetry/src/telemetry/store/utils/profileChunkProcessor.ts b/packages/telemetry/src/telemetry/store/utils/profileChunkProcessor.ts new file mode 100644 index 000000000..52bace29e --- /dev/null +++ b/packages/telemetry/src/telemetry/store/utils/profileChunkProcessor.ts @@ -0,0 +1,178 @@ +import type { + ProcessedProfileSample, + SentryProfileV2ChunkEvent, +} from "../../types.ts"; +import type { SentryProfileWithTraceMeta } from "../types.ts"; + +/** + * Represents a processed V2 profile chunk ready for storage and visualization. + * V2 chunks use absolute timestamps and are keyed by profiler_id. + */ +export type ProcessedProfileChunk = { + profiler_id: string; + chunk_id: string; + platform: string; + release?: string; + environment?: string; + samples: ProcessedProfileSample[]; + stacks: number[][]; + frames: SentryProfileV2ChunkEvent["profile"]["frames"]; + thread_metadata: SentryProfileV2ChunkEvent["profile"]["thread_metadata"]; + // Timestamps derived from samples + start_timestamp: number; + end_timestamp: number; +}; + +export interface ProfileChunkProcessingResult { + chunk: ProcessedProfileChunk; +} + +/** + * Processes a V2 profile chunk event (continuous profiling). + * V2 samples use absolute Unix timestamps, so no relative-to-start conversion needed. + */ +export function processProfileChunkEvent( + event: SentryProfileV2ChunkEvent, +): ProfileChunkProcessingResult { + const { profile, profiler_id, chunk_id, platform, release, environment } = + event; + + // Convert V2 samples to processed format + // V2 uses absolute Unix timestamps in seconds with microsecond precision + const processedSamples: ProcessedProfileSample[] = profile.samples.map( + (sample) => ({ + stack_id: sample.stack_id, + thread_id: sample.thread_id, + start_timestamp: sample.timestamp * 1000, // Convert to milliseconds for internal use + }), + ); + + // Sort samples by timestamp + processedSamples.sort((a, b) => a.start_timestamp - b.start_timestamp); + + // Calculate chunk time boundaries from samples + const start_timestamp = + processedSamples.length > 0 + ? processedSamples[0].start_timestamp + : Date.now(); + const end_timestamp = + processedSamples.length > 0 + ? processedSamples[processedSamples.length - 1].start_timestamp + : start_timestamp; + + // Reverse stacks to match the order expected by visualization (leaf first) + const reversedStacks = profile.stacks.map((s: number[]) => + Array.from(s).reverse(), + ); + + const chunk: ProcessedProfileChunk = { + profiler_id, + chunk_id, + platform, + release, + environment, + samples: processedSamples, + stacks: reversedStacks, + frames: profile.frames, + thread_metadata: profile.thread_metadata, + start_timestamp, + end_timestamp, + }; + + return { chunk }; +} + +/** + * Merges multiple profile chunks from the same profiler session into a single profile + * that can be used with existing profile visualization components. + */ +export function mergeChunksToProfile( + chunks: ProcessedProfileChunk[], + activeThreadId?: string, +): SentryProfileWithTraceMeta | null { + if (chunks.length === 0) return null; + + // Sort chunks by start timestamp + const sortedChunks = [...chunks].sort( + (a, b) => a.start_timestamp - b.start_timestamp, + ); + + // Merge frames and build frame index mapping + const mergedFrames: ProcessedProfileChunk["frames"] = []; + const frameIndexMaps: Map[] = []; + + for (const chunk of sortedChunks) { + const frameMap = new Map(); + for (let i = 0; i < chunk.frames.length; i++) { + // For simplicity, we add all frames and dedupe by reference + // A more sophisticated approach would dedupe by frame content + const newIndex = mergedFrames.length; + mergedFrames.push(chunk.frames[i]); + frameMap.set(i, newIndex); + } + frameIndexMaps.push(frameMap); + } + + // Merge stacks with remapped frame indices + const mergedStacks: number[][] = []; + const stackIndexMaps: Map[] = []; + + for (let chunkIdx = 0; chunkIdx < sortedChunks.length; chunkIdx++) { + const chunk = sortedChunks[chunkIdx]; + const frameMap = frameIndexMaps[chunkIdx]; + const stackMap = new Map(); + + for (let i = 0; i < chunk.stacks.length; i++) { + const remappedStack = chunk.stacks[i].map( + (frameIdx) => frameMap.get(frameIdx) ?? frameIdx, + ); + const newIndex = mergedStacks.length; + mergedStacks.push(remappedStack); + stackMap.set(i, newIndex); + } + stackIndexMaps.push(stackMap); + } + + // Merge samples with remapped stack indices + const mergedSamples: ProcessedProfileSample[] = []; + for (let chunkIdx = 0; chunkIdx < sortedChunks.length; chunkIdx++) { + const chunk = sortedChunks[chunkIdx]; + const stackMap = stackIndexMaps[chunkIdx]; + + for (const sample of chunk.samples) { + mergedSamples.push({ + ...sample, + stack_id: stackMap.get(sample.stack_id) ?? sample.stack_id, + }); + } + } + + // Sort all samples by timestamp + mergedSamples.sort((a, b) => a.start_timestamp - b.start_timestamp); + + // Merge thread metadata + const mergedThreadMetadata: ProcessedProfileChunk["thread_metadata"] = {}; + for (const chunk of sortedChunks) { + Object.assign(mergedThreadMetadata, chunk.thread_metadata); + } + + // Determine active thread ID - use provided or first thread with samples + const resolvedActiveThreadId = + activeThreadId || + (mergedSamples.length > 0 + ? mergedSamples[0].thread_id + : Object.keys(mergedThreadMetadata)[0] || "0"); + + // Calculate overall timestamp from first chunk + const timestamp = sortedChunks[0].start_timestamp; + + return { + platform: sortedChunks[0].platform, + thread_metadata: mergedThreadMetadata, + samples: mergedSamples, + frames: mergedFrames, + stacks: mergedStacks, + timestamp, + active_thread_id: resolvedActiveThreadId, + }; +} diff --git a/packages/telemetry/src/telemetry/store/utils/profileProcessor.ts b/packages/telemetry/src/telemetry/store/utils/profileProcessor.ts new file mode 100644 index 000000000..0d026a9ce --- /dev/null +++ b/packages/telemetry/src/telemetry/store/utils/profileProcessor.ts @@ -0,0 +1,88 @@ +import { graftProfileSpans } from "../../data/profiles.ts"; +import type { + ProfileSample, + SentryProfileTransactionInfo, + SentryProfileV1Event, + Trace, +} from "../../types.ts"; +import type { SentryProfileWithTraceMeta } from "../types.ts"; +import { relativeNsToTimestamp } from "../utils.ts"; + +export interface ProfileProcessingContext { + tracesById: Map; +} + +export interface ProfileProcessingResult { + profiles: Array<{ + traceId: string; + profile: SentryProfileWithTraceMeta; + }>; +} + +/** + * Processes a profile event and creates profile data for each transaction. + * Handles profile grafting onto existing traces if available. + */ +export function processProfileEvent( + event: SentryProfileV1Event, + context: ProfileProcessingContext, +): ProfileProcessingResult { + const { tracesById } = context; + const profiles: ProfileProcessingResult["profiles"] = []; + + // Ensure transactions array exists + if (!event.transactions) { + event.transactions = event.transaction ? [event.transaction] : []; + } + + for (const txn of event.transactions) { + // Skip string transaction IDs + if (typeof txn === "string") continue; + + const profileTxn = txn as SentryProfileTransactionInfo; + const trace = tracesById.get(profileTxn.trace_id); + + // Calculate timestamp + const timestamp = + trace && profileTxn.relative_start_ns != null + ? relativeNsToTimestamp( + trace.start_timestamp, + profileTxn.relative_start_ns, + ) + : event.timestamp; + + // Build profile data + const profile: SentryProfileWithTraceMeta = { + platform: event.platform, + thread_metadata: event.profile.thread_metadata, + samples: event.profile.samples.map((s: ProfileSample) => ({ + stack_id: s.stack_id, + thread_id: s.thread_id, + elapsed_since_start_ns: s.elapsed_since_start_ns, + start_timestamp: relativeNsToTimestamp( + timestamp, + s.elapsed_since_start_ns, + ), + })), + frames: event.profile.frames, + stacks: event.profile.stacks.map((s: number[]) => + Array.from(s).reverse(), + ), + timestamp, + active_thread_id: profileTxn.active_thread_id, + }; + + profiles.push({ + traceId: profileTxn.trace_id, + profile, + }); + + // Graft profile onto trace if it's ready + // Avoid grafting partial traces (where we mocked start_timestamp from event timestamp) + if (trace && trace.start_timestamp < trace.timestamp) { + graftProfileSpans(trace, profile); + } + } + + return { profiles }; +} diff --git a/packages/telemetry/src/telemetry/store/utils/spanProcessor.ts b/packages/telemetry/src/telemetry/store/utils/spanProcessor.ts new file mode 100644 index 000000000..2c737f451 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/utils/spanProcessor.ts @@ -0,0 +1,124 @@ +import type { + Sdk, + SentrySpanEvent, + SentryTransactionEvent, + Span, +} from "../../types.ts"; +import { initializeTrace } from "./traceInitializer.ts"; +import { + type TraceProcessingContext, + type TraceProcessingResult, + processTransactionEvent, + rebuildSpanTree, +} from "./traceProcessor.ts"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** Decode the span v2 container, or a legacy standalone span, without changing the raw envelope. */ +export function normalizeSpanItem( + payload: unknown, + sdk: Sdk, +): SentrySpanEvent[] { + if (!isRecord(payload)) return []; + const isContainer = "items" in payload; + if (isContainer && (payload.version !== 2 || !Array.isArray(payload.items))) + return []; + + const events: SentrySpanEvent[] = []; + for (const raw of isContainer ? (payload.items as unknown[]) : [payload]) { + if (!isRecord(raw)) continue; + const end = isContainer ? raw.end_timestamp : raw.timestamp; + if ( + typeof raw.trace_id !== "string" || + !raw.trace_id || + typeof raw.span_id !== "string" || + !raw.span_id || + typeof raw.start_timestamp !== "number" || + !Number.isFinite(raw.start_timestamp) || + typeof end !== "number" || + !Number.isFinite(end) + ) + continue; + + const data: Record = {}; + if (isContainer && isRecord(raw.attributes)) { + for (const [key, attribute] of Object.entries(raw.attributes)) { + if (isRecord(attribute) && "value" in attribute) + data[key] = attribute.value; + } + } else if (isRecord(raw.data)) { + Object.assign(data, raw.data); + } + + const op = isContainer ? data["sentry.op"] : raw.op; + const name = isContainer ? raw.name : raw.description; + const span: Span = { + trace_id: raw.trace_id, + span_id: raw.span_id, + parent_span_id: + typeof raw.parent_span_id === "string" ? raw.parent_span_id : undefined, + description: typeof name === "string" ? name : undefined, + op: typeof op === "string" ? op : undefined, + start_timestamp: raw.start_timestamp, + timestamp: end, + status: typeof raw.status === "string" ? raw.status : undefined, + data, + }; + + events.push({ + type: "span", + event_id: `${raw.trace_id}:${raw.span_id}`, + timestamp: end, + start_timestamp: raw.start_timestamp, + contexts: { + trace: { + trace_id: raw.trace_id, + span_id: raw.span_id, + parent_span_id: span.parent_span_id ?? undefined, + op: span.op ?? undefined, + status: span.status, + data, + }, + }, + sdk, + span, + is_segment: raw.is_segment === true, + }); + } + return events; +} + +export function processSpanEvent( + event: SentrySpanEvent, + context: TraceProcessingContext, +): TraceProcessingResult { + const trace = context.existingTrace ?? initializeTrace(event); + const span = { + ...event.span, + start_timestamp: event.span.start_timestamp, + timestamp: event.span.timestamp, + }; + + // A segment serves the same role as the root transaction in the existing UI. + // Child spans must not inflate the transaction count or replace its name. + if (event.is_segment) { + const transaction: SentryTransactionEvent = { + ...event, + type: "transaction", + transaction: span.description ?? "(unknown transaction)", + start_timestamp: + event.start_timestamp as SentryTransactionEvent["start_timestamp"], + contexts: { ...event.contexts, trace: event.contexts!.trace! }, + }; + return processTransactionEvent(transaction, { + ...context, + existingTrace: trace, + }); + } + + trace.spans.set(span.span_id, span); + rebuildSpanTree(trace); + return { trace, shouldUpdateTrace: true }; +} diff --git a/packages/telemetry/src/telemetry/store/utils/traceInitializer.ts b/packages/telemetry/src/telemetry/store/utils/traceInitializer.ts new file mode 100644 index 000000000..db7c219e2 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/utils/traceInitializer.ts @@ -0,0 +1,35 @@ +import type { + SentryEvent, + SentryTransactionEvent, + Span, + Trace, +} from "../../types.ts"; + +/** + * Creates a new trace object from an event's trace context. + * This is used when we receive an event with a trace_id but don't have a trace yet. + */ +export function initializeTrace(event: SentryEvent): Trace { + // Add guard to ensure contexts, trace context and trace_id exist + if (!event.contexts?.trace || !event.contexts.trace.trace_id) { + throw new Error("Event missing required trace context or trace_id"); + } + + const traceCtx = event.contexts.trace; + const timestamp = event.timestamp ?? 0; + + return { + ...traceCtx, + trace_id: traceCtx.trace_id!, + spans: new Map(), + spanTree: [] as Span[], + transactions: [] as SentryTransactionEvent[], + errors: 0, + start_timestamp: event.start_timestamp ?? timestamp, + timestamp, + status: traceCtx.status, + rootTransactionName: event.transaction || "(unknown transaction)", + rootTransaction: null, + profileGrafted: false, + }; +} diff --git a/packages/telemetry/src/telemetry/store/utils/traceProcessor.ts b/packages/telemetry/src/telemetry/store/utils/traceProcessor.ts new file mode 100644 index 000000000..7b8c0c3c7 --- /dev/null +++ b/packages/telemetry/src/telemetry/store/utils/traceProcessor.ts @@ -0,0 +1,216 @@ +import { graftProfileSpans } from "../../data/profiles.ts"; +import { log } from "../../logger.ts"; +import type { SentryTransactionEvent, Span, Trace } from "../../types.ts"; +import { compareSpans, groupSpans } from "../../utils/traces.ts"; +import type { SentryProfileWithTraceMeta } from "../types.ts"; +import type { ProcessedProfileChunk } from "./profileChunkProcessor.ts"; +import { mergeChunksToProfile } from "./profileChunkProcessor.ts"; + +export interface TraceProcessingContext { + existingTrace?: Trace; + profilesByTraceId: Map; + profileChunksByProfilerId?: Map; +} + +export interface TraceProcessingResult { + trace: Trace; + shouldUpdateTrace: boolean; + /** V2 profile merged from chunks, to be stored in profilesByTraceId */ + mergedProfile?: SentryProfileWithTraceMeta; +} + +/** + * Processes a transaction event and updates the trace data structure. + * Handles span tree construction, profile grafting, and trace metadata. + */ +export function processTransactionEvent( + event: SentryTransactionEvent, + context: TraceProcessingContext, +): TraceProcessingResult { + const traceCtx = event.contexts.trace; + const { existingTrace, profilesByTraceId, profileChunksByProfilerId } = + context; + + // Add guard to ensure trace_id exists + if (!traceCtx.trace_id) { + throw new Error("Transaction event missing required trace_id"); + } + + // Initialize or get existing trace + const trace: Trace = existingTrace ?? { + ...traceCtx, + trace_id: traceCtx.trace_id, + spans: new Map(), + spanTree: [] as Span[], + transactions: [] as SentryTransactionEvent[], + errors: 0, + start_timestamp: event.start_timestamp ?? event.timestamp, + timestamp: event.timestamp, + status: traceCtx.status, + rootTransactionName: event.transaction || "(unknown transaction)", + rootTransaction: null, + profileGrafted: false, + }; + + // Update trace timestamps + trace.start_timestamp = Math.min( + event.start_timestamp ?? event.timestamp, + trace.start_timestamp, + ); + trace.timestamp = Math.max(event.timestamp, trace.timestamp); + + // Legacy transactions and streamed segments can describe the same span. + const previousIndex = trace.transactions.findIndex( + (txn) => txn.contexts.trace.span_id === traceCtx.span_id, + ); + if (traceCtx.span_id && previousIndex !== -1) { + const previous = trace.transactions[previousIndex]; + trace.transactions[previousIndex] = { + ...previous, + ...event, + transaction: event.transaction ?? previous.transaction, + start_timestamp: Math.min( + previous.start_timestamp, + event.start_timestamp, + ), + timestamp: Math.max(previous.timestamp, event.timestamp), + contexts: { + ...previous.contexts, + ...event.contexts, + trace: { + ...previous.contexts.trace, + ...Object.fromEntries( + Object.entries(event.contexts.trace).filter( + ([, value]) => value !== undefined, + ), + ), + data: { + ...previous.contexts.trace.data, + ...event.contexts.trace.data, + }, + }, + }, + spans: Array.from( + new Map( + [...(previous.spans ?? []), ...(event.spans ?? [])].map((span) => [ + span.span_id, + span, + ]), + ).values(), + ), + }; + } else { + trace.transactions.push(event); + } + trace.transactions.sort(compareSpans); + + // Keep standalone spans received before this transaction/segment finished. + const spanMap: Map = new Map(trace.spans); + for (const txn of trace.transactions) { + const txnTrace = txn.contexts.trace; + if (!txnTrace || !txnTrace.span_id || !txnTrace.trace_id) { + continue; + } + + spanMap.set(txnTrace.span_id, { + ...txnTrace, + span_id: txnTrace.span_id, + trace_id: txnTrace.trace_id, + tags: txn?.tags, + start_timestamp: txn.start_timestamp, + timestamp: txn.timestamp, + description: traceCtx.description || txn.transaction, + transaction: txn, + }); + + // Add child spans + if (txn.spans) { + for (const span of txn.spans) { + spanMap.set(span.span_id, { + ...span, + timestamp: span.timestamp, + start_timestamp: span.start_timestamp, + }); + } + } + } + + trace.spans = spanMap; + rebuildSpanTree(trace); + + // Graft profile data if available (V1 profiles keyed by trace_id) + const profileForTrace = profilesByTraceId.get(trace.trace_id); + if (profileForTrace) { + graftProfileSpans(trace, profileForTrace); + } + + // Try V2 continuous profiling: look for profiler_id in transaction context + // V2 uses contexts.profile.profiler_id to link profiles + let mergedProfile: SentryProfileWithTraceMeta | undefined; + if (!trace.profileGrafted && profileChunksByProfilerId) { + const profileContext = event.contexts?.profile as + | { profiler_id?: string } + | undefined; + const profilerId = profileContext?.profiler_id; + + if (profilerId) { + const chunks = profileChunksByProfilerId.get(profilerId); + if (chunks && chunks.length > 0) { + // Get the active thread ID from trace context data if available + const activeThreadId = traceCtx.data?.["thread.id"] as + | string + | undefined; + mergedProfile = + mergeChunksToProfile(chunks, activeThreadId) ?? undefined; + if (mergedProfile) { + graftProfileSpans(trace, mergedProfile); + } + } + } + } + + return { + trace, + shouldUpdateTrace: true, + mergedProfile, + }; +} + +/** Rebuild from real spans so late parents replace placeholders and children are not appended twice. */ +export function rebuildSpanTree(trace: Trace): void { + trace.spans = new Map( + Array.from(trace.spans) + .filter(([, span]) => span.op !== "orphan") + .map(([id, span]) => [id, { ...span, children: undefined }]), + ); + trace.spanTree = groupSpans(trace.spans); +} + +/** + * Updates trace metadata based on its transactions. + * Determines root transaction and handles orphan traces. + */ +export function updateTraceMetadata(trace: Trace): void { + const roots = trace.transactions.filter( + (e) => !e.contexts.trace.parent_span_id, + ); + + if (roots.length === 1) { + trace.rootTransaction = roots[0]; + trace.rootTransactionName = roots[0].transaction || "(unknown transaction)"; + } else if (roots.length > 1) { + trace.rootTransactionName = "(multiple root transactions)"; + } else if (trace.transactions.length > 0) { + // Orphan trace: no root transaction, but has child transactions + log( + `[Telemetry] Orphan trace detected (trace_id: ${trace.trace_id}). ` + + `Using first transaction "${trace.transactions[0].transaction}" as fallback.`, + ); + // use the first transcation for orphan traces + trace.rootTransaction = trace.transactions[0]; + trace.rootTransactionName = + trace.transactions[0].transaction || "(orphan transaction)"; + } else { + trace.rootTransactionName = "(missing root transaction)"; + } +} diff --git a/packages/telemetry/src/telemetry/topology.spec.ts b/packages/telemetry/src/telemetry/topology.spec.ts new file mode 100644 index 000000000..590b60e5b --- /dev/null +++ b/packages/telemetry/src/telemetry/topology.spec.ts @@ -0,0 +1,44 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "../sdk/core.ts"; + +test("a self-parented span remains visible as a tree root", async () => { + const telemetry = createTelemetryStore(); + try { + const result = await telemetry.ingestEnvelope([ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "trace", + span_id: "self", + parent_span_id: "self", + start_timestamp: 1, + end_timestamp: 2, + }, + { + trace_id: "trace", + span_id: "child", + parent_span_id: "self", + start_timestamp: 1.1, + end_timestamp: 1.5, + }, + ], + }, + ], + ], + ]); + expect(result.status).toBe("processed"); + const trace = telemetry.getTrace("trace")!; + expect(trace.spans.size).toBe(2); + expect(trace.spanTree.map((span) => span.span_id)).toEqual(["self"]); + expect(trace.spanTree[0].children?.map((span) => span.span_id)).toEqual([ + "child", + ]); + } finally { + telemetry.dispose(); + } +}); diff --git a/packages/telemetry/src/telemetry/types.ts b/packages/telemetry/src/telemetry/types.ts new file mode 100644 index 000000000..061d6ee63 --- /dev/null +++ b/packages/telemetry/src/telemetry/types.ts @@ -0,0 +1,495 @@ +import type { + EnvelopeItem, + EventEnvelopeHeaders, + Measurements, + SerializedLog, +} from "@sentry/core"; + +export type TraceId = string; +export type SpanId = string; + +export type EventAttachment = { + header: EnvelopeItem[0]; + data: string; +}; + +export type FrameVars = { + [key: string]: string; +}; + +export type EventFrame = { + filename?: string; + abs_path?: string; + function?: string; + module?: string; + lineno?: number; + colno?: number; + pre_context?: string[]; + post_context?: string[]; + context_line?: string; + vars?: FrameVars; + instruction_addr?: string; + in_app?: boolean; +}; + +export type EventStacktrace = { + frames: EventFrame[]; +}; + +export type EventExceptionValue = { + type: string; + value: string; + stacktrace?: EventStacktrace; +}; + +export type EventException = + | { + values: EventExceptionValue[]; + value: undefined; + } + | { + values: undefined; + value: EventExceptionValue; + }; + +export type Breadcrumb = { + data?: Record; + message?: string; + category: string; + timestamp: string; + type: string; +}; + +export type Context = Record; + +type CommonEventAttrs = { + // not always present, but we are forcing it in EventCache + event_id: string; + timestamp: number; + message?: SentryFormattedMessage; + breadcrumbs?: Breadcrumb[] | { values: Breadcrumb[] }; + transaction?: string; + environment?: string; + platform?: string; + server_name?: string; + release?: string; + start_timestamp?: number; + contexts?: Contexts; + tags?: Tags; + extra?: Context; + request?: Record | string>; + modules?: Record; + sdk?: Sdk; + measurements?: Measurements; + attachments?: EventAttachment[]; + // Inferred source type for distinguishing browser/server/mobile events + __sourceType?: "browser" | "server" | "mobile"; +}; + +// Note: For some reason the `sentry/core` module doesn't have these additional properties +// in `EventEnvelopeHeaders['trace']` but they are present in the actual events. +// Follow up? +export type TraceContext = EventEnvelopeHeaders["trace"] & { + span_id?: string; + status?: string; + description?: string; + parent_span_id?: string; + data?: Record; + op?: string; +}; + +export type FeedbackContext = { + message: string; + name?: string; + contact_email?: string; + source?: string; + url?: string; + associated_event_id?: string; + replay_id?: string; +}; + +export type Contexts = { + runtime?: Context; + browser?: Context; + trace?: TraceContext; + feedback?: FeedbackContext; +} & { + [key: string]: Context | TraceContext | FeedbackContext | undefined; +}; + +export type Tags = { + [key: string]: string; +}; + +export type SentryFormattedMessage = + | string + | { + formatted: string; + params?: []; + }; + +export type SentryErrorEvent = CommonEventAttrs & { + type?: "error" | "event" | "message" | "default"; + level?: "fatal" | "error" | "warning" | "info" | "debug"; + exception: EventException; +}; + +export type Span = { + trace_id?: TraceId; + span_id: SpanId; + parent_span_id?: string | null; + op?: string | null; + description?: string | null; + start_timestamp: number; + tags?: Tags | null; + timestamp: number; + status?: string; + transaction?: SentryTransactionEvent; + children?: Span[]; + data?: Record; +}; + +export type SentryTransactionEvent = CommonEventAttrs & { + type: "transaction"; + spans?: Span[]; + start_timestamp: number; + contexts: Contexts & { + trace: TraceContext; + }; +}; + +// Standalone spans are normalized at the envelope boundary. Only segment spans +// are represented as transactions; child spans remain individual spans. +export type SentrySpanEvent = CommonEventAttrs & { + type: "span"; + span: Span; + is_segment: boolean; +}; + +export type ProfileSample = { + elapsed_since_start_ns: string; + stack_id: number; + thread_id: string; +}; + +export type ProcessedProfileSample = { + start_timestamp: number; + stack_id: number; + thread_id: string; +}; + +export type SentryProfile = { + samples: ProfileSample[]; + stacks: number[][]; + frames: EventFrame[]; + platform?: string; + thread_metadata?: Record< + string, + { + name?: string; + priority?: number; + } + >; +}; + +export type SentryProcessedProfile = Omit & { + samples: ProcessedProfileSample[]; +}; + +export type SentryProfileWithTraceMeta = SentryProcessedProfile & { + timestamp: number; + active_thread_id: string; +}; + +export type AggregateCallData = { + name: string; + totalTime: number; + samples: number; + traceIds: Set; +}; + +export type SentryDeviceInfo = { + architecture: string; + is_emulator?: boolean; + locale?: string; + manufacturer?: string; + model?: string; +}; + +export type SentryOSInfo = { + name: string; + version: string; + build_number?: string; +}; + +export type SentryProfileTransactionInfo = { + name: string; + id: string; + trace_id: string; + active_thread_id: string; + relative_start_ns?: string; + relative_end_ns?: string; +}; + +export type SentryProfileV1Event = CommonEventAttrs & { + type: "profile"; + device: SentryDeviceInfo; + os: SentryOSInfo; + transactions?: Array; + transaction?: SentryProfileTransactionInfo; + version: "1"; + profile: SentryProfile; +}; + +// V2 Profile (Continuous Profiling) Types +export type ProfileV2Sample = { + timestamp: number; // Unix timestamp in seconds with microseconds precision + stack_id: number; + thread_id: string; +}; + +export type ProfileV2MeasurementValue = { + timestamp: number; + value: number; +}; + +export type ProfileV2Measurement = { + unit: string; + values: ProfileV2MeasurementValue[]; +}; + +export type SentryProfileV2 = { + samples: ProfileV2Sample[]; + stacks: number[][]; + frames: EventFrame[]; + thread_metadata: Record< + string, + { + name?: string; + priority?: number; + } + >; +}; + +export type SentryProfileV2ChunkEvent = Omit< + Partial, + "measurements" +> & + Pick & { + type: "profile_chunk"; + version: "2"; + profiler_id: string; // Links chunks from same profiler session + chunk_id: string; // Unique ID for this chunk + platform: string; + release?: string; + environment?: string; + client_sdk?: { + name: string; + version: string; + }; + debug_meta?: { + images?: Array<{ + debug_id?: string; + image_addr?: string; + type?: string; + image_size?: number; + code_file?: string; + }>; + }; + measurements?: Record; + profile: SentryProfileV2; + }; + +export type SentryLogEventItem = SerializedLog & { + id: string; // Need to have a unique id for each log + severity_number: number; + sdk: string | undefined; +}; + +export type SentryLogEvent = CommonEventAttrs & { + type: "log"; + items: Array; +}; + +export type SentryFeedbackEvent = CommonEventAttrs & { + type: "feedback"; + contexts: Contexts & { + feedback: FeedbackContext; + }; +}; + +export type SentryEvent = + | SentryErrorEvent + | SentryTransactionEvent + | SentrySpanEvent + | SentryProfileV1Event + | SentryProfileV2ChunkEvent + | SentryLogEvent + | SentryFeedbackEvent; + +export type Trace = TraceContext & { + trace_id: string; + transactions: SentryTransactionEvent[]; + errors: number; + start_timestamp: number; + timestamp: number; + status?: string; + rootTransaction: SentryTransactionEvent | null; + rootTransactionName: string; + spans: Map; + spanTree: Span[]; + profileGrafted: boolean; +}; + +export type Sdk = { + name: string; + version: string; + lastSeen: number; +}; + +export type SentryEventWithPerformanceData = Omit< + SentryEvent, + "measurements" +> & { + measurements: Record< + string, + { + value: number; + unit: string; + } + > & { + "score.total": { + value: number; + unit: string; + }; + "score.fcp": { + value: number; + unit: string; + }; + "score.lcp": { + value: number; + unit: string; + }; + "score.fid": { + value: number; + unit: string; + }; + "score.cls": { + value: number; + unit: string; + }; + "score.ttfb": { + value: number; + unit: string; + }; + "score.weight.fcp": { + value: number; + unit: string; + }; + "score.weight.lcp": { + value: number; + unit: string; + }; + "score.weight.fid": { + value: number; + unit: string; + }; + "score.weight.cls": { + value: number; + unit: string; + }; + "score.weight.ttfb": { + value: number; + unit: string; + }; + }; +}; + +export type MetricScoreProps = { + fcpScore: number; + lcpScore: number; + clsScore: number; + fidScore: number; + ttfbScore: number; +}; +export type MetricWeightsProps = { + fcp: number; + lcp: number; + cls: number; + fid: number; + ttfb: number; +}; + +export type AIToolCall = { + toolCallId: string; + toolName: string; + args: Record; + result?: Record | string; + state?: string; + step?: number; +}; + +// Content can be string (v1) or array of content blocks (v2) +export type AIContentBlock = { + type: string; + text?: string; +}; + +export type AIMessageContent = string | AIContentBlock[]; + +export type AIMessage = { + role: string; + content: AIMessageContent; + toolInvocations?: AIToolCall[]; + parts?: unknown[]; +}; + +export type AIPrompt = { + system?: string; + messages?: AIMessage[]; +}; + +export type AIResponse = { + finishReason?: string; + text?: string; + toolCalls?: AIToolCall[]; +}; + +export type AIMetadata = { + modelId?: string; + modelProvider?: string; + functionId?: string; + metadata: Record; + maxRetries?: number; + maxSteps?: number; + promptTokens?: number; + completionTokens?: number; +}; + +export type AITrace = { + id: string; // root span_id, also used as traceId for display + name: string; // root span description or op for primary display + operation: string; + timestamp: number; + durationMs: number; + tokensDisplay: string; // prompt/completion or N/A + promptTokens?: number; + completionTokens?: number; + hasToolCall: boolean; + rawSpan: Span; + metadata: AIMetadata; + prompt?: AIPrompt; + response?: AIResponse; + toolCalls: AIToolCall[]; +}; + +export type AILibraryHandler = { + id: string; + name: string; + canHandleSpan: (span: Span) => boolean; + extractRootSpans: (spans: Span[]) => Span[]; + processTrace: (rootSpan: Span) => AITrace; + getDisplayTitle: (trace: AITrace) => string; + getTypeBadge: (trace: AITrace) => string; + getTokensDisplay: (trace: AITrace) => string; +}; diff --git a/packages/telemetry/src/telemetry/utils/profileUtils.ts b/packages/telemetry/src/telemetry/utils/profileUtils.ts new file mode 100644 index 000000000..bffe1f823 --- /dev/null +++ b/packages/telemetry/src/telemetry/utils/profileUtils.ts @@ -0,0 +1,16 @@ +import type { EventFrame } from "../types.ts"; + +const _FUNCTION_NAME_FROM_FRAME_CACHE = new WeakMap(); +export function getFunctionNameFromFrame(frame: EventFrame): string { + let result = _FUNCTION_NAME_FROM_FRAME_CACHE.get(frame); + if (!result) { + const module = + frame.module || frame.filename || frame.abs_path || ""; + const functionName = frame.function || ""; + const lineNo = frame.lineno ? `:${frame.lineno}` : ""; + const colNo = frame.lineno && frame.colno ? `:${frame.colno}` : ""; + result = `${module}@${functionName}${lineNo}${colNo}`; + _FUNCTION_NAME_FROM_FRAME_CACHE.set(frame, result); + } + return result; +} diff --git a/packages/telemetry/src/telemetry/utils/sdkToPlatform.ts b/packages/telemetry/src/telemetry/utils/sdkToPlatform.ts new file mode 100644 index 000000000..e7cd4215f --- /dev/null +++ b/packages/telemetry/src/telemetry/utils/sdkToPlatform.ts @@ -0,0 +1,15 @@ +/** + * Normalizes SDK name to a platform. Can be as specific or unspecific as we support different platforms and SDKs. + */ +export function sdkToPlatform(name: string) { + if (name.includes("javascript.astro")) return "javascript.astro"; + if (name.includes("javascript")) return "javascript"; + if (name.includes("java")) return "java"; + if (name.includes("python")) return "python"; + if (name.includes("php.laravel")) return "php.laravel"; + if (name.includes("php.symfony")) return "php.symfony"; + if (name.includes("php")) return "php"; + if (name.includes("ruby")) return "ruby"; + if (name.includes("dotnet")) return "dotnet"; + return "unknown"; +} diff --git a/packages/telemetry/src/telemetry/utils/sentry.ts b/packages/telemetry/src/telemetry/utils/sentry.ts new file mode 100644 index 000000000..e6b046d76 --- /dev/null +++ b/packages/telemetry/src/telemetry/utils/sentry.ts @@ -0,0 +1,62 @@ +import { + ERROR_EVENT_TYPES, + FEEDBACK_EVENT_TYPES, + LOG_EVENT_TYPES, + PROFILE_EVENT_TYPES, + TRACE_EVENT_TYPES, +} from "../constants/sentry.ts"; +import type { + SentryErrorEvent, + SentryEvent, + SentryFeedbackEvent, + SentryLogEvent, + SentryProfileV1Event, + SentryProfileV2ChunkEvent, + SentryTransactionEvent, +} from "../types.ts"; + +export function isErrorEvent(event: SentryEvent): event is SentryErrorEvent { + return ( + (!event.type || ERROR_EVENT_TYPES.has(event.type)) && + Boolean((event as SentryErrorEvent).exception) + ); +} + +export function isProfileEvent( + event: SentryEvent, +): event is SentryProfileV1Event { + return ( + !!event.type && + PROFILE_EVENT_TYPES.has(event.type) && + (event as SentryProfileV1Event).version === "1" + ); +} + +export function isProfileChunkEvent( + event: SentryEvent, +): event is SentryProfileV2ChunkEvent { + return ( + event.type === "profile_chunk" && + (event as SentryProfileV2ChunkEvent).version === "2" + ); +} + +export function isTraceEvent( + event: SentryEvent, +): event is SentryTransactionEvent { + return !!event.type && TRACE_EVENT_TYPES.has(event.type); +} + +export function isLogEvent(event: SentryEvent): event is SentryLogEvent { + return !!event.type && LOG_EVENT_TYPES.has(event.type); +} + +export function isFeedbackEvent( + event: SentryEvent, +): event is SentryFeedbackEvent { + return ( + !!event.type && + FEEDBACK_EVENT_TYPES.has(event.type) && + Boolean((event as SentryFeedbackEvent).contexts?.feedback) + ); +} diff --git a/packages/telemetry/src/telemetry/utils/stringifyValue.test.ts b/packages/telemetry/src/telemetry/utils/stringifyValue.test.ts new file mode 100644 index 000000000..618d46ac2 --- /dev/null +++ b/packages/telemetry/src/telemetry/utils/stringifyValue.test.ts @@ -0,0 +1,14 @@ +import { expect, test } from "vitest"; +import { stringifyValue } from "./stringifyValue.ts"; + +test("formats scalar attributes and structured wire values explicitly", () => { + expect(stringifyValue("text")).toBe("text"); + expect(stringifyValue(0)).toBe("0"); + expect(stringifyValue(false)).toBe("false"); + expect(stringifyValue(null)).toBe("null"); + expect(stringifyValue({ text: "hello" })).toBe('{"text":"hello"}'); + expect(stringifyValue(["one", "two"])).toBe('["one","two"]'); + const circular: { self?: unknown } = {}; + circular.self = circular; + expect(stringifyValue(circular)).toBe(""); +}); diff --git a/packages/telemetry/src/telemetry/utils/stringifyValue.ts b/packages/telemetry/src/telemetry/utils/stringifyValue.ts new file mode 100644 index 000000000..dba65da36 --- /dev/null +++ b/packages/telemetry/src/telemetry/utils/stringifyValue.ts @@ -0,0 +1,9 @@ +/** Preserve scalar text and serialize structured wire values without default object coercion. */ +export function stringifyValue(value: unknown): string { + if (typeof value !== "object" || value === null) return String(value); + try { + return JSON.stringify(value) ?? ""; + } catch { + return ""; + } +} diff --git a/packages/telemetry/src/telemetry/utils/traces.ts b/packages/telemetry/src/telemetry/utils/traces.ts new file mode 100644 index 000000000..874305ef4 --- /dev/null +++ b/packages/telemetry/src/telemetry/utils/traces.ts @@ -0,0 +1,107 @@ +import { stringifyValue } from "./stringifyValue.ts"; +import { log } from "../logger.ts"; +import type { Span, Trace } from "../types.ts"; + +// mutates spans in place and adds children, as well as returns the top level tree +export function groupSpans(spans: Map): Span[] { + const tree: Span[] = []; + + // need to sort root(s) first + const sortedSpans = Array.from(spans.values()).sort((a, b) => { + const parentComp = (a.parent_span_id ? 1 : 0) - (b.parent_span_id ? 1 : 0); + return parentComp === 0 ? compareSpans(a, b) : parentComp; + }); + + for (const span of sortedSpans) { + let parent = span && getParentOfSpan(span, spans, sortedSpans); + + span.children ??= []; + if (parent) { + parent.children ??= []; + parent.children.push(span); + } else if (span.parent_span_id && span.parent_span_id !== span.span_id) { + const parentParent = sortedSpans.find((s) => !s.parent_span_id); + if (!parentParent) { + log( + `Root span (${span.parent_span_id}) for span (${span.span_id}). Creating orphan.`, + ); + } else { + log( + `Creating orphan for parent (${span.parent_span_id}) for span (${span.span_id})`, + ); + } + parent = { + trace_id: span.trace_id, + span_id: span.parent_span_id, + parent_span_id: parentParent ? parentParent.span_id : null, + op: "orphan", + description: "missing or unknown parent span", + children: [span], + start_timestamp: span.start_timestamp, + timestamp: span.timestamp, + status: "unknown", + }; + spans.set(parent.span_id, parent); + // sortedSpans.splice(spanIdx, 0, parent); + if (parentParent) { + parentParent.children ??= []; + parentParent.children.push(parent); + } else { + tree.push(parent); + } + } else { + tree.push(span); + } + spans.set(span.span_id, span); + } + + return tree; +} + +function getParentOfSpan( + span: Span, + idLookup: Map, + allSpans: Span[], +): Span | undefined { + if (!span.parent_span_id || span.parent_span_id === span.span_id) { + return undefined; + } + + return ( + idLookup.get(span.parent_span_id) || + allSpans.find((s) => s.span_id === span.parent_span_id) + ); +} + +export function compareSpans( + a: { start_timestamp: number }, + b: { start_timestamp: number }, +): number { + return a.start_timestamp - b.start_timestamp; +} + +export function getRootTransactionMethod(trace: Trace) { + // Fallback to first transaction if no root transaction (e.g., orphan traces) + const transaction = trace.rootTransaction || trace.transactions[0]; + const method = stringifyValue( + transaction?.contexts?.trace.data?.method || + transaction?.request?.method || + "", + ); + return method; +} + +export function getRootTransactionName(trace: Trace) { + const method = getRootTransactionMethod(trace); + // For orphan traces without a root, use the first transaction's name + let name = trace.rootTransactionName; + if (!trace.rootTransaction && trace.transactions.length > 0) { + name = trace.transactions[0].transaction || trace.rootTransactionName; + } + + // Strip method prefix if present + const finalName = + method && name.startsWith(method) ? name.slice(method.length + 1) : name; + + return finalName; +} diff --git a/packages/telemetry/src/telemetry/utils/webVitals.ts b/packages/telemetry/src/telemetry/utils/webVitals.ts new file mode 100644 index 000000000..67e4ba833 --- /dev/null +++ b/packages/telemetry/src/telemetry/utils/webVitals.ts @@ -0,0 +1,200 @@ +import type { SentryEvent } from "../types.ts"; + +const SQRT_2 = Math.sqrt(2); + +// Gauss error function +function erf(x: number): number { + const a1 = 0.254829592; + const a2 = -0.284496736; + const a3 = 1.421413741; + const a4 = -1.453152027; + const a5 = 1.061405429; + const p = 0.3275911; + + const sign = x < 0 ? -1 : 1; + const absX = Math.abs(x); + + const t = 1.0 / (1.0 + p * absX); + const y = + 1.0 - + ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX); + + return sign * y; +} + +// Sigma function for CDF score calculation +export function calculateCdfSigma(p10: number, p50: number): number { + return ( + Math.abs(Math.log(p10) - Math.log(p50)) / (SQRT_2 * 0.9061938024368232) + ); +} + +// Calculates a log-normal CDF score based on a log-normal with a specific p10 and p50 +export function calculateCdfScore( + value: number, + p10: number, + p50: number, +): number { + return ( + 0.5 * + (1.0 - + erf( + (Math.log(value) - Math.log(p50)) / + (SQRT_2 * calculateCdfSigma(p50, p10)), + )) + ); +} + +interface ScoreComponent { + measurement: string; + weight: number; + p10: number; + p50: number; + optional?: boolean; +} + +interface Profile { + name: string; + scoreComponents: ScoreComponent[]; + condition?: { + op: string; + name: string; + value: string | number; + }; +} + +export interface PerformanceScoreConfig { + profiles: Profile[]; +} + +// export function getPropertyByPath(obj: Record, path: string): any { +// const keys = path.split('.'); +// let result = obj; + +// for (const key of keys) { +// if (result && result[key]) { +// result = result[key]; +// } else { +// return undefined; +// } +// } + +// return result; +// } + +// export function matchCondition(condition: { op: string; name: string; value: string | number }, event: SentryEvent): boolean { +// const { op, name, value } = condition; + +// if (op === 'eq') { +// const eventValue = getPropertyByPath({ event }, name); +// return eventValue === value; +// } + +// return false; +// } + +export function normalizePerformanceScore( + event: SentryEvent, + performanceScore: PerformanceScoreConfig | undefined, +): void { + if (!performanceScore || event.type === "profile_chunk") { + return; + } + + for (const profile of performanceScore.profiles) { + // TODO: To check a matching condition as done in relay. Skipped right now because Browser type is not coming in Event data + // const condition = profile.condition; + // if (condition && !matchCondition(condition, event)) { + // continue; + // } + + const measurements = event.measurements; + + if (measurements) { + let shouldAddTotal = false; + + // TODO: To calculate performance score if all required metric fields are present in event measurement data. + // if ( + // profile.scoreComponents.some(c => { + // return ( + // !Object.prototype.hasOwnProperty.call(measurements, c.measurement) && + // Math.abs(c.weight) >= Number.EPSILON && + // !c.optional + // ); + // }) + // ) { + // break; + // } + + let scoreTotal = 0.0; + let weightTotal = 0.0; + + for (const component of profile.scoreComponents) { + if ( + component.optional && + !Object.prototype.hasOwnProperty.call( + measurements, + component.measurement, + ) + ) { + continue; + } + + weightTotal += component.weight; + } + + if (Math.abs(weightTotal) < Number.EPSILON) { + break; + } + + for (const component of profile.scoreComponents) { + let normalizedComponentWeight = 0.0; + + if ( + Object.prototype.hasOwnProperty.call( + measurements, + component.measurement, + ) + ) { + normalizedComponentWeight = component.weight / weightTotal; + const value = measurements[component.measurement].value; + const cdf = calculateCdfScore( + Math.max(0.0, value), + component.p10, + component.p50, + ); + const componentScore = cdf * normalizedComponentWeight; + scoreTotal += componentScore; + shouldAddTotal = true; + + measurements[`score.${component.measurement}`] = { + value: cdf, + unit: "ratio", + }; + } else { + measurements[`score.${component.measurement}`] = { + value: 0, + unit: "ratio", + }; + } + + measurements[`score.weight.${component.measurement}`] = { + value: normalizedComponentWeight, + unit: "ratio", + }; + } + + if ( + shouldAddTotal && + !Object.prototype.hasOwnProperty.call(measurements, "score.total") + ) { + measurements["score.total"] = { + value: scoreTotal, + unit: "ratio", + }; + } + + break; + } + } +} diff --git a/packages/telemetry/src/telemetry/uuid.ts b/packages/telemetry/src/telemetry/uuid.ts new file mode 100644 index 000000000..78a62a691 --- /dev/null +++ b/packages/telemetry/src/telemetry/uuid.ts @@ -0,0 +1,9 @@ +export function generateUuidv4() { + let dt = new Date().getTime(); + return "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, (c) => { + let rnd = Math.random() * 16; //random number in range 0 to 16 + rnd = ((dt + rnd) % 16) | 0; + dt = Math.floor(dt / 16); + return (c === "x" ? rnd : (rnd & 0x3) | 0x8).toString(16); + }); +} diff --git a/packages/telemetry/tests/types/sdk-core.ts b/packages/telemetry/tests/types/sdk-core.ts new file mode 100644 index 000000000..36b3db696 --- /dev/null +++ b/packages/telemetry/tests/types/sdk-core.ts @@ -0,0 +1,116 @@ +import { + type IngestResult, + type ProcessedEvent, + type QuerySummary, + SDK_CAPABILITIES, + type TelemetryReader, + createTelemetryReader, + createTelemetryStore, + serializeSnapshot, + serializeSummaries, + serializeTrace, +} from "@sentry/telemetry"; + +const telemetry = createTelemetryStore(); +const queries: readonly QuerySummary[] = telemetry.getDatabaseQuerySummaries({ + traceId: "trace", + includeSpans: true, +}); +const events: readonly ProcessedEvent[] = telemetry.getEvents({ + traceId: "trace", + spanId: "span", +}); +const envelope = telemetry.getEnvelope("envelope"); +if (envelope) { + const [header, items] = envelope; + const sdk = header.sdk; + for (const [itemHeader, payload] of items) { + const type: string = itemHeader.type; + void type; + void payload; + } + void sdk; +} +// @ts-expect-error State storage is private. +void telemetry.store; +// @ts-expect-error Public events are read-only. +events.push(events[0]); +// @ts-expect-error Nested maps cannot be mutated through the public type. +telemetry.getTrace("trace")?.spans.clear(); +// @ts-expect-error Duration units are explicit in the public contract. +void queries[0].totalTime; +const normalizedTimestamp: number = events[0].timestamp; +void normalizedTimestamp; +void queries[0].totalDurationMs; +void queries[0].averageDurationMs; + +const reader: TelemetryReader = createTelemetryReader(() => + telemetry.getSnapshot(), +); +// @ts-expect-error Read-only consumers cannot ingest. +void reader.ingestEnvelope; +const compact = reader.getDatabaseQuerySummaries(); +// @ts-expect-error Full spans require includeSpans: true. +const full: readonly QuerySummary[] = compact; +void full; +const page = reader.readPage("events", { + fromMs: 0, + toMs: 10000, + order: "desc", + limit: 20, +}); +if (page.status === "ok") { + const events: readonly ProcessedEvent[] = page.items; + void events; + const cursor: string | undefined = page.nextCursor; + void cursor; +} else { + const code: "stale_cursor" | "invalid_cursor" | "invalid_query" = page.code; + void code; +} +const snapshot = serializeSnapshot(reader.getSnapshot()); +const version: 1 = snapshot.schemaVersion; +const unit: "unix-milliseconds" = SDK_CAPABILITIES.timestampUnit; +const trace = reader.getTrace("trace"); +if (trace) JSON.stringify(serializeTrace(trace)); +JSON.stringify(snapshot); +JSON.stringify(serializeSummaries(compact)); +JSON.stringify(serializeSummaries(reader.getProfileFunctionSummaries())); +// @ts-expect-error Export full span graphs through serializeTrace instead. +serializeSummaries(queries); +void version; +void unit; +async function receipts() { + const results: readonly IngestResult[] = await telemetry.ingestEnvelopes( + [null], + { + signal: new AbortController().signal, + }, + ); + const receipt = await telemetry.ingestEnvelope(null, { deliveryId: "retry" }); + const retained: boolean = receipt.retained; + const revision: number = receipt.revision; + const duplicates: number = receipt.counts.duplicateEvents; + // @ts-expect-error Receipts are immutable. + receipt.counts.events = 0; + if (receipt.status === "ignored") { + // @ts-expect-error Reasons are a literal union. + const reason: "arbitrary-text" = receipt.reason; + void reason; + } + if (receipt.status === "invalid") { + const severity: "warning" | "error" = receipt.diagnostics[0].severity; + const index: number | undefined = receipt.diagnostics[0].itemIndex; + void severity; + void index; + } + const resized: Promise = telemetry.setRetention({ maxEnvelopes: 10 }); + await resized; + await telemetry.drain(); + void results; + void retained; + void revision; + void duplicates; +} +void receipts; +telemetry.dispose(); diff --git a/packages/telemetry/tests/types/sdk-node.ts b/packages/telemetry/tests/types/sdk-node.ts new file mode 100644 index 000000000..296fac730 --- /dev/null +++ b/packages/telemetry/tests/types/sdk-node.ts @@ -0,0 +1,42 @@ +import type { IngestResult, TelemetryReader } from "@sentry/telemetry"; +import { + type TelemetryPayload, + createTelemetryPipeline, +} from "@sentry/telemetry/node"; + +const pipeline = createTelemetryPipeline({ + maxPending: 4, + maxPayloadBytes: 1024, + maxDecodedBytes: 2048, + decodedHistory: { maxBytes: 4096, maxEntries: 3 }, + retention: { maxEnvelopes: 10, maxBytes: 8192, maxAgeMs: 1000 }, + enrichStacktrace: async (_event, { signal }) => { + signal.throwIfAborted(); + }, +}); +const reader: TelemetryReader = pipeline; +const input: TelemetryPayload = { + body: new Uint8Array(), + deliveryId: "request-1", + signal: new AbortController().signal, +}; +const receipt: Promise = pipeline.ingestPayload(input); +const batch: Promise = pipeline.ingestPayloads([ + input, +]); +const decoded = pipeline.getDecodedPayload("envelope"); +const body: Uint8Array | undefined = decoded?.body; +// @ts-expect-error Serialized payload ingestion has an explicit name. +void pipeline.ingest; +// @ts-expect-error Decoded history has an explicit name. +void pipeline.getRawPayload; +const resized: Promise = pipeline.setRetention({ + maxEnvelopes: 10, + maxAgeMs: 1000, +}); +void resized; +void reader; +void receipt; +void batch; +void body; +pipeline.dispose(); diff --git a/packages/telemetry/tests/types/tsconfig.sdk-core.json b/packages/telemetry/tests/types/tsconfig.sdk-core.json new file mode 100644 index 000000000..f7e6581c4 --- /dev/null +++ b/packages/telemetry/tests/types/tsconfig.sdk-core.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023", "DOM"], + "types": [], + "strict": true, + "noEmit": true, + "skipLibCheck": false + }, + "files": ["sdk-core.ts"] +} diff --git a/packages/telemetry/tests/types/tsconfig.sdk-node.json b/packages/telemetry/tests/types/tsconfig.sdk-node.json new file mode 100644 index 000000000..828ff9216 --- /dev/null +++ b/packages/telemetry/tests/types/tsconfig.sdk-node.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.sdk-core.json", + "compilerOptions": { "types": ["node"] }, + "files": ["sdk-node.ts"] +} diff --git a/packages/telemetry/tsconfig.build.json b/packages/telemetry/tsconfig.build.json new file mode 100644 index 000000000..5773ee319 --- /dev/null +++ b/packages/telemetry/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "declaration": true, + "sourceMap": true, + "outDir": "dist", + "rootDir": "src" + } +} diff --git a/packages/telemetry/tsconfig.json b/packages/telemetry/tsconfig.json new file mode 100644 index 000000000..b740e92f9 --- /dev/null +++ b/packages/telemetry/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": [ + "ES2023", + "DOM" + ], + "types": [ + "node" + ], + "strict": true, + "skipLibCheck": false, + "rewriteRelativeImportExtensions": true, + "noEmit": true + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "src/**/*.test.ts", + "src/**/*.spec.ts" + ] +} diff --git a/packages/telemetry/vitest.config.ts b/packages/telemetry/vitest.config.ts new file mode 100644 index 000000000..fa3ecf9b1 --- /dev/null +++ b/packages/telemetry/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["src/**/*.{test,spec}.ts"], + testTimeout: 30000, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce1f08907..b0ffc46e7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -352,6 +352,9 @@ importers: '@radix-ui/react-tooltip': specifier: ^1.4.1 version: 1.4.1(@types/react-dom@19.3.0(@types/react@19.3.0))(@types/react@19.3.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) + '@sentry/telemetry': + specifier: workspace:* + version: link:../../packages/telemetry '@shikijs/core': specifier: ^3.23.0 version: 3.23.0 @@ -416,9 +419,6 @@ importers: specifier: 'catalog:' version: 1.4.2(typescript@5.8.3) devDependencies: - '@spotlightjs/spotlight': - specifier: ^4.11.7 - version: 4.11.8(@cfworker/json-schema@4.1.1)(hono-rate-limiter@0.4.2(hono@4.13.8)) '@testing-library/react': specifier: ^16.3.2 version: 16.3.3(@testing-library/dom@10.4.2)(@types/react-dom@19.3.0(@types/react@19.3.0))(@types/react@19.3.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -488,12 +488,12 @@ importers: '@sentry/symbolic': specifier: 13.7.0 version: 13.7.0 + '@sentry/telemetry': + specifier: workspace:* + version: link:../telemetry '@sentry/toolkit-core': specifier: workspace:* version: link:../toolkit-core - '@spotlightjs/spotlight': - specifier: ^4.11.7 - version: 4.11.8(@cfworker/json-schema@4.1.1)(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1))(hono-rate-limiter@0.4.2(hono@4.13.13)) '@stricli/auto-complete': specifier: ^1.2.8 version: 1.3.0 @@ -1021,6 +1021,34 @@ importers: specifier: 'catalog:' version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@22.16.0)(@vitest/coverage-v8@4.1.11)(jsdom@28.1.0)(msw@2.10.2(@types/node@22.16.0)(typescript@5.9.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) + packages/telemetry: + dependencies: + '@sentry/core': + specifier: ^10.63.0 + version: 10.63.0(patch_hash=e663994979ff877a26ab6d4dea5968fbaee4ccdfdeb85623983535a572678940) + uuidv7: + specifier: ^1.0.2 + version: 1.2.1 + zustand: + specifier: ^5.0.3 + version: 5.0.16(@types/react@19.1.8)(react@19.2.8)(use-sync-external-store@1.5.0(react@19.2.8)) + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 22.16.0 + '@types/react': + specifier: 'catalog:' + version: 19.1.8 + react: + specifier: 'catalog:' + version: 19.2.8 + typescript: + specifier: 'catalog:' + version: 5.8.3 + vitest: + specifier: 'catalog:' + version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@22.16.0)(@vitest/coverage-v8@4.1.11)(jsdom@28.1.0)(msw@2.10.2(@types/node@22.16.0)(typescript@5.8.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) + packages/toolkit-core: dependencies: '@sentry/search-syntax': @@ -2583,12 +2611,6 @@ packages: hono-rate-limiter: ^0.4.2 zod: ^3.25.0 || ^4.0.0 - '@hono/node-server@1.19.17': - resolution: {integrity: sha512-dSneS5qhiauZWGDCeK4o695Xd9nUNjviSZCMQrj10eetr8Uln1ucn6bbphOM6UynAMMtNIzZNSpL9vnASJwrPQ==} - engines: {node: '>=18.14.1'} - peerDependencies: - hono: ^4 - '@hono/node-server@2.1.1': resolution: {integrity: sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==} engines: {node: '>=20'} @@ -4856,11 +4878,6 @@ packages: '@speed-highlight/core@1.2.7': resolution: {integrity: sha512-0dxmVj4gxg3Jg879kvFS/msl4s9F3T9UXC1InxgOf7t5NvcPD97u/WTA5vL/IxWHMn7qSxBozqrnnE2wvl1m8g==} - '@spotlightjs/spotlight@4.11.8': - resolution: {integrity: sha512-653j6YqzaWoTMGXB1l8jJonQuc/ssa3CDqP6RIyiclekm3UwGJVFtPkccfHll8nJiv0qJDmYXlJlrbmyFPfQjw==} - engines: {node: '>=20'} - hasBin: true - '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -5398,9 +5415,6 @@ packages: resolution: {integrity: sha512-MxT4XZL7pzLHpuvhDKdMaQHMGGkJDLluKBLsbstn+8wv9sWcFT6h+0ve9qkml95amVTZtZV83gQe2hY+ojgHLg==} hasBin: true - anser@2.3.5: - resolution: {integrity: sha512-vcZjxvvVoxTeR5XBNJB38oTu/7eDCZlwdz32N1eNgpyPF7j/Z7Idf+CUwQOkKKpJ7RJyjxgLHCM7vdIK0iCNMQ==} - ansi-escapes@7.0.0: resolution: {integrity: sha512-GdYO7a61mR0fOlAsvC9/rIHf7L96sBc6dEWzeOu+KAea5bZyQRPIpojrVoI4AXGJS/ycu/fBTdLrUkA4ODrvjw==} engines: {node: '>=18'} @@ -6389,9 +6403,6 @@ packages: fast-fifo@1.3.2: resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} - fast-fuzzy@1.12.0: - resolution: {integrity: sha512-sXxGgHS+ubYpsdLnvOvJ9w5GYYZrtL9mkosG3nfuD446ahvoWEsSKBP7ieGmWIKVLnaxRDgUJkZMdxRgA2Ni+Q==} - fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -6597,9 +6608,6 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} - graphemesplit@2.6.0: - resolution: {integrity: sha512-rG9w2wAfkpg0DILa1pjnjNfucng3usON360shisqIMUBw/87pojcBSrHmeE4UwryAuBih7g8m1oilf5/u8EWdQ==} - graphql@16.14.2: resolution: {integrity: sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} @@ -6706,10 +6714,6 @@ packages: resolution: {integrity: sha512-CQ46U0ZkAGmbT/4UxdzzGJpacP2IeKgY4a5/tOI9AABbpOMfK739wfDXmv1usCk+3RkKj1hQy4/fjhiwa2xlrA==} engines: {node: '>=16.9.0'} - hono@4.13.8: - resolution: {integrity: sha512-/Gng7NfoykZl2pjukW5Z6+8Yxm3BPRf86GTbQnt0SbySkvax4fyL4H3HhY1cCpBGmiW9XDRFzRV+CXK2W8QudQ==} - engines: {node: '>=16.9.0'} - hookable@5.5.3: resolution: {integrity: sha512-Yc+BQe8SvoXH1643Qez1zqLRmbA5rCL+sSmk6TVos0LWVfNIB7PGncdlId77WzLGSIB5KaWgTaNTs2lNVEI6VQ==} @@ -7083,9 +7087,6 @@ packages: resolution: {integrity: sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==} engines: {node: '>= 8'} - launch-editor@2.14.1: - resolution: {integrity: sha512-QWBrQsMpH7gPr965dsKD/3cKWiNoTjpATQf++Xq63N6sKRGMwlVXz41O1IZTMfZQgBctD/K5Zt06+/I6pP6+HA==} - lightningcss-android-arm64@1.32.0: resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} engines: {node: '>= 12.0.0'} @@ -7273,10 +7274,6 @@ packages: resolution: {integrity: sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==} engines: {node: '>=18'} - logfmt@1.4.0: - resolution: {integrity: sha512-p1Ow0C2dDJYaQBhRHt+HVMP6ELuBm4jYSYNHPMfz0J5wJ9qA6/7oBOlBZBfT1InqguTYcvJzNea5FItDxTcbyw==} - hasBin: true - longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -7372,10 +7369,6 @@ packages: math-random@2.0.1: resolution: {integrity: sha512-oIEbWiVDxDpl5tIF4S6zYS9JExhh3bun3uLb3YAinHPTlRtW4g1S66LtJrJ4Npq8dgIa8CLK5iPVah5n4n0s2w==} - mcp-proxy@5.12.5: - resolution: {integrity: sha512-Vawdc8vi36fXxKCaDpluRvbGcmrUXJdvXcDhkh30HYsws8XqX2rWPBflZpavzeS+6SwijRFV7g+9ypQRJZlrEQ==} - hasBin: true - mdast-util-definitions@6.0.0: resolution: {integrity: sha512-scTllyX6pnYNZH/AIp/0ePz6s4cZtARxImwoPJ7kS42n+MnVsI4XbnG6d4ibehRIldYMWM2LD7ImQblVhUejVQ==} @@ -7903,9 +7896,6 @@ packages: resolution: {integrity: sha512-XTUaK0hXMCu2jszWE584JGQT7y284TmMV9l/HX3rnG5uo3rHI/uHU56XTyyyPFjeWEBxECbAi0CaFDJOONtG0Q==} hasBin: true - pako@0.2.9: - resolution: {integrity: sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==} - parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -8667,9 +8657,6 @@ packages: space-separated-tokens@2.0.2: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} - split@0.2.10: - resolution: {integrity: sha512-e0pKq+UUH2Xq/sXbYpZBZc3BawsfDZ7dgv+JtRTUPNcvF5CMR4Y9cvJqkMY0MoxWzTHvZuz1beg6pNEKlszPiQ==} - stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -8834,9 +8821,6 @@ packages: resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==} engines: {node: '>=18'} - through@2.3.8: - resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} - tiktoken@1.0.22: resolution: {integrity: sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==} @@ -9035,9 +9019,6 @@ packages: unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} - unicode-trie@2.0.0: - resolution: {integrity: sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ==} - unicorn-magic@0.3.0: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} @@ -9658,6 +9639,24 @@ packages: zod@4.6.7: resolution: {integrity: sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==} + zustand@5.0.16: + resolution: {integrity: sha512-Uv7e327ihlCfayfja3hDc0M64Upev1LWBKpRobdK4NyovDDPeeX4imWXc/wzC1yc6fkYPoAY/ncKVPAs2jrtQw==} + engines: {node: '>=12.20.0'} + peerDependencies: + '@types/react': '>=18.0.0' + immer: '>=9.0.6' + react: '>=18.0.0' + use-sync-external-store: '>=1.2.0' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + use-sync-external-store: + optional: true + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -11012,22 +11011,6 @@ snapshots: '@fontsource/ibm-plex-mono@5.3.0': {} - '@hono/mcp@0.2.5(@modelcontextprotocol/sdk@1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5))(hono-rate-limiter@0.4.2(hono@4.13.13))(hono@4.13.8)(zod@4.6.5)': - dependencies: - '@modelcontextprotocol/sdk': 1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5) - hono: 4.13.8 - hono-rate-limiter: 0.4.2(hono@4.13.13) - pkce-challenge: 5.0.1 - zod: 4.6.5 - - '@hono/mcp@0.2.5(@modelcontextprotocol/sdk@1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5))(hono-rate-limiter@0.4.2(hono@4.13.8))(hono@4.13.8)(zod@4.6.5)': - dependencies: - '@modelcontextprotocol/sdk': 1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5) - hono: 4.13.8 - hono-rate-limiter: 0.4.2(hono@4.13.8) - pkce-challenge: 5.0.1 - zod: 4.6.5 - '@hono/mcp@0.2.5(@modelcontextprotocol/sdk@1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.7))(hono-rate-limiter@0.4.2(hono@4.13.13))(hono@4.13.13)(zod@4.6.7)': dependencies: '@modelcontextprotocol/sdk': 1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.7) @@ -11036,10 +11019,6 @@ snapshots: pkce-challenge: 5.0.1 zod: 4.6.7 - '@hono/node-server@1.19.17(hono@4.13.8)': - dependencies: - hono: 4.13.8 - '@hono/node-server@2.1.1(hono@4.13.13)': dependencies: hono: 4.13.13 @@ -11494,30 +11473,6 @@ snapshots: dependencies: zod: 4.6.7 - '@modelcontextprotocol/sdk@1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5)': - dependencies: - '@hono/node-server': 2.1.1(hono@4.13.13) - ajv: 8.20.0 - ajv-formats: 3.0.1(ajv@8.20.0) - content-type: 1.0.5 - cors: 2.8.6 - cross-spawn: 7.0.6 - eventsource: 3.0.7 - eventsource-parser: 3.1.1 - express: 5.3.0 - express-rate-limit: 8.7.1(express@5.3.0) - hono: 4.13.13 - jose: 6.2.12 - json-schema-typed: 8.0.2 - pkce-challenge: 5.0.1 - raw-body: 3.0.2 - zod: 4.6.5 - zod-to-json-schema: 3.25.2(zod@4.6.5) - optionalDependencies: - '@cfworker/json-schema': 4.1.1 - transitivePeerDependencies: - - supports-color - '@modelcontextprotocol/sdk@1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.7)': dependencies: '@hono/node-server': 2.1.1(hono@4.13.13) @@ -13128,60 +13083,6 @@ snapshots: '@speed-highlight/core@1.2.7': {} - '@spotlightjs/spotlight@4.11.8(@cfworker/json-schema@4.1.1)(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1))(hono-rate-limiter@0.4.2(hono@4.13.13))': - dependencies: - '@hono/mcp': 0.2.5(@modelcontextprotocol/sdk@1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5))(hono-rate-limiter@0.4.2(hono@4.13.13))(hono@4.13.8)(zod@4.6.5) - '@hono/node-server': 1.19.17(hono@4.13.8) - '@jridgewell/trace-mapping': 0.3.31 - '@modelcontextprotocol/sdk': 1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5) - '@sentry/core': 10.63.0(patch_hash=e663994979ff877a26ab6d4dea5968fbaee4ccdfdeb85623983535a572678940) - '@sentry/node': 10.65.0(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)) - anser: 2.3.5 - chalk: 5.6.2 - eventsource: 4.1.1 - fast-fuzzy: 1.12.0 - hono: 4.13.8 - launch-editor: 2.14.1 - logfmt: 1.4.0 - mcp-proxy: 5.12.5 - semver: 7.8.5 - uuidv7: 1.2.1 - yaml: 2.9.0 - zod: 4.6.5 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@opentelemetry/core' - - '@opentelemetry/exporter-trace-otlp-http' - - hono-rate-limiter - - supports-color - - '@spotlightjs/spotlight@4.11.8(@cfworker/json-schema@4.1.1)(hono-rate-limiter@0.4.2(hono@4.13.8))': - dependencies: - '@hono/mcp': 0.2.5(@modelcontextprotocol/sdk@1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5))(hono-rate-limiter@0.4.2(hono@4.13.8))(hono@4.13.8)(zod@4.6.5) - '@hono/node-server': 1.19.17(hono@4.13.8) - '@jridgewell/trace-mapping': 0.3.31 - '@modelcontextprotocol/sdk': 1.32.0(@cfworker/json-schema@4.1.1)(zod@4.6.5) - '@sentry/core': 10.63.0(patch_hash=e663994979ff877a26ab6d4dea5968fbaee4ccdfdeb85623983535a572678940) - '@sentry/node': 10.65.0(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)) - anser: 2.3.5 - chalk: 5.6.2 - eventsource: 4.1.1 - fast-fuzzy: 1.12.0 - hono: 4.13.8 - launch-editor: 2.14.1 - logfmt: 1.4.0 - mcp-proxy: 5.12.5 - semver: 7.8.5 - uuidv7: 1.2.1 - yaml: 2.9.0 - zod: 4.6.5 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@opentelemetry/core' - - '@opentelemetry/exporter-trace-otlp-http' - - hono-rate-limiter - - supports-color - '@standard-schema/spec@1.1.0': {} '@standard-schema/utils@0.3.0': {} @@ -13385,7 +13286,7 @@ snapshots: '@types/node-fetch@2.6.13': dependencies: - '@types/node': 24.13.5 + '@types/node': 22.20.3 form-data: 4.0.6 '@types/node@18.19.130': @@ -13440,7 +13341,7 @@ snapshots: '@types/sax@1.2.7': dependencies: - '@types/node': 24.13.5 + '@types/node': 22.20.3 '@types/semver@7.8.0': {} @@ -13526,6 +13427,15 @@ snapshots: msw: 2.10.2(@types/node@22.16.0)(typescript@5.8.3) vite: 8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.20.3)(yaml@2.9.0) + '@vitest/mocker@4.1.11(msw@2.10.2(@types/node@22.16.0)(typescript@5.8.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0))': + dependencies: + '@vitest/spy': 4.1.11 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + msw: 2.10.2(@types/node@22.16.0)(typescript@5.8.3) + vite: 8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0) + '@vitest/mocker@4.1.11(msw@2.10.2(@types/node@22.16.0)(typescript@5.9.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.11 @@ -13719,8 +13629,6 @@ snapshots: dependencies: process-ancestry: 0.1.0 - anser@2.3.5: {} - ansi-escapes@7.0.0: dependencies: environment: 1.1.0 @@ -14822,10 +14730,6 @@ snapshots: fast-fifo@1.3.2: {} - fast-fuzzy@1.12.0: - dependencies: - graphemesplit: 2.6.0 - fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -15047,11 +14951,6 @@ snapshots: graceful-fs@4.2.11: {} - graphemesplit@2.6.0: - dependencies: - js-base64: 3.7.8 - unicode-trie: 2.0.0 - graphql@16.14.2: {} gray-matter@4.0.3: @@ -15286,14 +15185,8 @@ snapshots: dependencies: hono: 4.13.13 - hono-rate-limiter@0.4.2(hono@4.13.8): - dependencies: - hono: 4.13.8 - hono@4.13.13: {} - hono@4.13.8: {} - hookable@5.5.3: {} html-encoding-sniffer@6.0.0: @@ -15653,11 +15546,6 @@ snapshots: klona@2.0.6: {} - launch-editor@2.14.1: - dependencies: - picocolors: 1.1.1 - shell-quote: 1.10.0 - lightningcss-android-arm64@1.32.0: optional: true @@ -15808,11 +15696,6 @@ snapshots: strip-ansi: 7.1.2 wrap-ansi: 9.0.2 - logfmt@1.4.0: - dependencies: - split: 0.2.10 - through: 2.3.8 - longest-streak@3.1.0: {} loose-envify@1.4.0: @@ -15891,8 +15774,6 @@ snapshots: math-random@2.0.1: {} - mcp-proxy@5.12.5: {} - mdast-util-definitions@6.0.0: dependencies: '@types/mdast': 4.0.4 @@ -16875,8 +16756,6 @@ snapshots: '@pagefind/windows-arm64': 1.5.2 '@pagefind/windows-x64': 1.5.2 - pako@0.2.9: {} - parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -17871,10 +17750,6 @@ snapshots: space-separated-tokens@2.0.2: {} - split@0.2.10: - dependencies: - through: 2.3.8 - stack-utils@2.0.6: dependencies: escape-string-regexp: 2.0.0 @@ -18051,8 +17926,6 @@ snapshots: throttleit@2.1.0: {} - through@2.3.8: {} - tiktoken@1.0.22: {} tiny-inflate@1.0.3: {} @@ -18250,11 +18123,6 @@ snapshots: dependencies: pathe: 2.0.3 - unicode-trie@2.0.0: - dependencies: - pako: 0.2.9 - tiny-inflate: 1.0.3 - unicorn-magic@0.3.0: {} unified@11.0.5: @@ -18581,6 +18449,36 @@ snapshots: transitivePeerDependencies: - msw + vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@22.16.0)(@vitest/coverage-v8@4.1.11)(jsdom@28.1.0)(msw@2.10.2(@types/node@22.16.0)(typescript@5.8.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)): + dependencies: + '@vitest/expect': 4.1.11 + '@vitest/mocker': 4.1.11(msw@2.10.2(@types/node@22.16.0)(typescript@5.8.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) + '@vitest/pretty-format': 4.1.11 + '@vitest/runner': 4.1.11 + '@vitest/snapshot': 4.1.11 + '@vitest/spy': 4.1.11 + '@vitest/utils': 4.1.11 + es-module-lexer: 2.3.2 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.7 + std-env: 4.0.0 + tinybench: 2.9.0 + tinyexec: 1.3.1 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.0 + vite: 8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 22.16.0 + '@vitest/coverage-v8': 4.1.11(vitest@4.1.11) + jsdom: 28.1.0 + transitivePeerDependencies: + - msw + vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@22.16.0)(@vitest/coverage-v8@4.1.11)(jsdom@28.1.0)(msw@2.10.2(@types/node@22.16.0)(typescript@5.9.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.11 @@ -19046,10 +18944,6 @@ snapshots: dependencies: zod: 4.6.7 - zod-to-json-schema@3.25.2(zod@4.6.5): - dependencies: - zod: 4.6.5 - zod-to-json-schema@3.25.2(zod@4.6.7): dependencies: zod: 4.6.7 @@ -19060,4 +18954,10 @@ snapshots: zod@4.6.7: {} + zustand@5.0.16(@types/react@19.1.8)(react@19.2.8)(use-sync-external-store@1.5.0(react@19.2.8)): + optionalDependencies: + '@types/react': 19.1.8 + react: 19.2.8 + use-sync-external-store: 1.5.0(react@19.2.8) + zwitch@2.0.4: {} From 01a2f760516885262e2b900de9e4dbd08408d0e3 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Wed, 28 Oct 2026 16:59:46 +0530 Subject: [PATCH 02/15] fix(spotlight): Rename SDK and address processing review Publish the shared package as @sentry/spotlight and update workspace, consumer, and release paths. Preserve retained trace views, incomplete span summaries, AI metadata, and thread-aware profile processing with regression coverage. Co-Authored-By: Codex --- .craft.yml | 16 +- .github/workflows/local.yml | 2 +- .github/workflows/merge-jobs.yml | 18 +- .github/workflows/release.yml | 8 +- apps/local/README.md | 2 +- apps/local/package.json | 10 +- apps/local/src/lib/inspection.ts | 4 +- apps/local/src/lib/performance.ts | 6 +- apps/local/src/lib/profiles.ts | 4 +- .../src/lib/telemetry-regressions.test.ts | 293 ++++++++++++++++++ apps/local/src/lib/telemetry-store.ts | 25 +- apps/local/src/lib/trace-model.ts | 47 +-- apps/local/src/test-fixtures/trace-groups.ts | 2 +- docs/README.md | 2 +- docs/cli/local-control-plane.md | 2 +- packages/cli/package.json | 6 +- .../src/commands/local/telemetry-buffer.ts | 4 +- packages/{telemetry => spotlight}/LICENSE | 0 packages/{telemetry => spotlight}/README.md | 48 +-- .../continuous_profiling/profile_chunk_1.txt | 0 .../continuous_profiling/transaction.txt | 0 .../_fixtures/envelope_span_v2.txt | 0 .../examples/frontend.ts | 2 +- .../examples/lifecycle.ts | 4 +- .../examples/react.tsx | 2 +- .../examples/reader.ts | 2 +- .../examples/server.ts | 2 +- .../examples/tsconfig.json | 0 .../{telemetry => spotlight}/package.json | 4 +- .../scripts/check-package.mjs | 6 +- .../{telemetry => spotlight}/src/sdk/core.ts | 0 .../src/sdk/node.spec.ts | 0 .../{telemetry => spotlight}/src/sdk/node.ts | 0 .../src/telemetry/ai-regressions.spec.ts | 229 ++++++++++++++ .../src/telemetry/ai/aiLibraries.ts | 0 .../src/telemetry/ai/genAI.ts | 35 ++- .../src/telemetry/ai/inspection.ts | 0 .../src/telemetry/ai/vercelAISDK.ts | 62 +++- .../src/telemetry/attachments.spec.ts | 0 .../src/telemetry/capabilities.ts | 0 .../src/telemetry/client.spec.ts | 0 .../src/telemetry/constants/performance.ts | 0 .../src/telemetry/constants/sentry.ts | 0 .../src/telemetry/contracts.ts | 0 .../src/telemetry/data/profiles.test.ts | 275 ++++++++++++++++ .../src/telemetry/data/profiles.ts | 40 ++- .../src/telemetry/envelope/codec.spec.ts | 0 .../src/telemetry/envelope/codec.ts | 0 .../src/telemetry/envelope/payload.ts | 0 .../src/telemetry/envelope/source.ts | 0 .../src/telemetry/envelope/userAgent.ts | 0 .../src/telemetry/index.ts | 0 .../src/telemetry/inspection-models.spec.ts | 0 .../src/telemetry/internal/engine.ts | 0 .../src/telemetry/internal/envelopes.ts | 0 .../src/telemetry/internal/readonly.spec.ts | 0 .../src/telemetry/internal/readonly.ts | 0 .../src/telemetry/internal/results.ts | 0 .../src/telemetry/internal/retention.ts | 0 .../src/telemetry/internal/types.ts | 0 .../src/telemetry/internal/wireValues.ts | 0 .../src/telemetry/lifecycle.spec.ts | 0 .../src/telemetry/log-traces.spec.ts | 0 .../src/telemetry/logger.ts | 0 .../src/telemetry/profiles/model.ts | 0 .../src/telemetry/public.ts | 0 .../src/telemetry/queries.ts | 0 .../src/telemetry/reader.spec.ts | 0 .../src/telemetry/reader.ts | 0 .../src/telemetry/selectors/ai.ts | 0 .../src/telemetry/selectors/database.ts | 0 .../src/telemetry/selectors/index.ts | 0 .../src/telemetry/selectors/profiles.test.ts | 84 +++++ .../src/telemetry/selectors/profiles.ts | 24 +- .../src/telemetry/selectors/resources.ts | 0 .../src/telemetry/selectors/selectors.test.ts | 26 +- .../src/telemetry/selectors/spans.ts | 5 +- .../src/telemetry/selectors/vitalPages.ts | 0 .../src/telemetry/selectors/webVitals.ts | 0 .../src/telemetry/serialization.spec.ts | 0 .../src/telemetry/serialization.ts | 0 .../spotlight/src/telemetry/store/helpers.ts | 16 + .../telemetry/store/slices/envelopesSlice.ts | 0 .../src/telemetry/store/slices/eventsSlice.ts | 0 .../src/telemetry/store/slices/logsSlice.ts | 0 .../telemetry/store/slices/profilesSlice.ts | 0 .../src/telemetry/store/slices/sdksSlice.ts | 0 .../src/telemetry/store/slices/sharedSlice.ts | 0 .../store/slices/subscriptionsSlice.ts | 0 .../src/telemetry/store/slices/tracesSlice.ts | 0 .../src/telemetry/store/types.ts | 0 .../src/telemetry/store/utils.ts | 0 .../src/telemetry/store/utils/logProcessor.ts | 0 .../store/utils/profileChunkProcessor.ts | 0 .../telemetry/store/utils/profileProcessor.ts | 0 .../telemetry/store/utils/spanProcessor.ts | 0 .../telemetry/store/utils/traceInitializer.ts | 0 .../telemetry/store/utils/traceProcessor.ts | 0 .../src/telemetry/topology.spec.ts | 0 .../src/telemetry/types.ts | 0 .../src/telemetry/utils/profileUtils.ts | 0 .../src/telemetry/utils/sdkToPlatform.ts | 0 .../src/telemetry/utils/sentry.ts | 0 .../telemetry/utils/stringifyValue.test.ts | 0 .../src/telemetry/utils/stringifyValue.ts | 0 .../src/telemetry/utils/traces.ts | 0 .../src/telemetry/utils/webVitals.ts | 0 .../src/telemetry/uuid.ts | 0 .../tests/types/sdk-core.ts | 2 +- .../tests/types/sdk-node.ts | 4 +- .../tests/types/tsconfig.sdk-core.json | 0 .../tests/types/tsconfig.sdk-node.json | 0 .../tsconfig.build.json | 0 .../{telemetry => spotlight}/tsconfig.json | 0 .../{telemetry => spotlight}/vitest.config.ts | 0 .../telemetry/src/telemetry/store/helpers.ts | 8 - pnpm-lock.yaml | 10 +- 117 files changed, 1170 insertions(+), 171 deletions(-) rename packages/{telemetry => spotlight}/LICENSE (100%) rename packages/{telemetry => spotlight}/README.md (94%) rename packages/{telemetry => spotlight}/_fixtures/continuous_profiling/profile_chunk_1.txt (100%) rename packages/{telemetry => spotlight}/_fixtures/continuous_profiling/transaction.txt (100%) rename packages/{telemetry => spotlight}/_fixtures/envelope_span_v2.txt (100%) rename packages/{telemetry => spotlight}/examples/frontend.ts (97%) rename packages/{telemetry => spotlight}/examples/lifecycle.ts (96%) rename packages/{telemetry => spotlight}/examples/react.tsx (91%) rename packages/{telemetry => spotlight}/examples/reader.ts (97%) rename packages/{telemetry => spotlight}/examples/server.ts (97%) rename packages/{telemetry => spotlight}/examples/tsconfig.json (100%) rename packages/{telemetry => spotlight}/package.json (95%) rename packages/{telemetry => spotlight}/scripts/check-package.mjs (96%) rename packages/{telemetry => spotlight}/src/sdk/core.ts (100%) rename packages/{telemetry => spotlight}/src/sdk/node.spec.ts (100%) rename packages/{telemetry => spotlight}/src/sdk/node.ts (100%) create mode 100644 packages/spotlight/src/telemetry/ai-regressions.spec.ts rename packages/{telemetry => spotlight}/src/telemetry/ai/aiLibraries.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/ai/genAI.ts (93%) rename packages/{telemetry => spotlight}/src/telemetry/ai/inspection.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/ai/vercelAISDK.ts (91%) rename packages/{telemetry => spotlight}/src/telemetry/attachments.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/capabilities.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/client.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/constants/performance.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/constants/sentry.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/contracts.ts (100%) create mode 100644 packages/spotlight/src/telemetry/data/profiles.test.ts rename packages/{telemetry => spotlight}/src/telemetry/data/profiles.ts (86%) rename packages/{telemetry => spotlight}/src/telemetry/envelope/codec.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/envelope/codec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/envelope/payload.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/envelope/source.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/envelope/userAgent.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/index.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/inspection-models.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/internal/engine.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/internal/envelopes.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/internal/readonly.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/internal/readonly.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/internal/results.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/internal/retention.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/internal/types.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/internal/wireValues.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/lifecycle.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/log-traces.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/logger.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/profiles/model.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/public.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/queries.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/reader.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/reader.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/selectors/ai.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/selectors/database.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/selectors/index.ts (100%) create mode 100644 packages/spotlight/src/telemetry/selectors/profiles.test.ts rename packages/{telemetry => spotlight}/src/telemetry/selectors/profiles.ts (61%) rename packages/{telemetry => spotlight}/src/telemetry/selectors/resources.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/selectors/selectors.test.ts (86%) rename packages/{telemetry => spotlight}/src/telemetry/selectors/spans.ts (86%) rename packages/{telemetry => spotlight}/src/telemetry/selectors/vitalPages.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/selectors/webVitals.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/serialization.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/serialization.ts (100%) create mode 100644 packages/spotlight/src/telemetry/store/helpers.ts rename packages/{telemetry => spotlight}/src/telemetry/store/slices/envelopesSlice.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/slices/eventsSlice.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/slices/logsSlice.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/slices/profilesSlice.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/slices/sdksSlice.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/slices/sharedSlice.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/slices/subscriptionsSlice.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/slices/tracesSlice.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/types.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/utils.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/utils/logProcessor.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/utils/profileChunkProcessor.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/utils/profileProcessor.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/utils/spanProcessor.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/utils/traceInitializer.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/store/utils/traceProcessor.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/topology.spec.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/types.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/utils/profileUtils.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/utils/sdkToPlatform.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/utils/sentry.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/utils/stringifyValue.test.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/utils/stringifyValue.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/utils/traces.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/utils/webVitals.ts (100%) rename packages/{telemetry => spotlight}/src/telemetry/uuid.ts (100%) rename packages/{telemetry => spotlight}/tests/types/sdk-core.ts (99%) rename packages/{telemetry => spotlight}/tests/types/sdk-node.ts (91%) rename packages/{telemetry => spotlight}/tests/types/tsconfig.sdk-core.json (100%) rename packages/{telemetry => spotlight}/tests/types/tsconfig.sdk-node.json (100%) rename packages/{telemetry => spotlight}/tsconfig.build.json (100%) rename packages/{telemetry => spotlight}/tsconfig.json (100%) rename packages/{telemetry => spotlight}/vitest.config.ts (100%) delete mode 100644 packages/telemetry/src/telemetry/store/helpers.ts diff --git a/.craft.yml b/.craft.yml index 1261f2cc0..0f71edc4f 100644 --- a/.craft.yml +++ b/.craft.yml @@ -111,20 +111,20 @@ workspaces: id: "@sentry/mcp-server" includeNames: '/^sentry-mcp-server-\d.*\.tgz$/' - packages/telemetry: + packages/spotlight: changelogPolicy: none - preReleaseCommand: bash -c 'cd packages/telemetry && npm version "$CRAFT_NEW_VERSION" --no-git-tag-version' - releaseBranchPrefix: release/telemetry + preReleaseCommand: bash -c 'cd packages/spotlight && npm version "$CRAFT_NEW_VERSION" --no-git-tag-version' + releaseBranchPrefix: release/spotlight artifactProvider: name: github config: artifacts: Post-merge tasks: - - "/^sentry-telemetry-.*$/" + - "/^sentry-spotlight-.*$/" targets: - name: github - tagPrefix: "telemetry@" - includeNames: '/^sentry-telemetry-\d.*\.tgz$/' + tagPrefix: "spotlight@" + includeNames: '/^sentry-spotlight-\d.*\.tgz$/' - name: npm - id: "@sentry/telemetry" - includeNames: '/^sentry-telemetry-\d.*\.tgz$/' + id: "@sentry/spotlight" + includeNames: '/^sentry-spotlight-\d.*\.tgz$/' diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml index ae0cacbee..d11772226 100644 --- a/.github/workflows/local.yml +++ b/.github/workflows/local.yml @@ -4,7 +4,7 @@ on: pull_request: paths: - "apps/local/**" - - "packages/telemetry/**" + - "packages/spotlight/**" - "packages/cli/src/commands/local/**" - "packages/cli/test/commands/local/**" - "packages/cli/test/e2e/local.test.ts" diff --git a/.github/workflows/merge-jobs.yml b/.github/workflows/merge-jobs.yml index d1c1f067b..238e73f61 100644 --- a/.github/workflows/merge-jobs.yml +++ b/.github/workflows/merge-jobs.yml @@ -2,7 +2,7 @@ name: Post-merge tasks on: push: - branches: ["main", "release/*", "release/mcp/**", "release/telemetry/**"] + branches: ["main", "release/*", "release/mcp/**", "release/spotlight/**"] workflow_dispatch: jobs: @@ -38,13 +38,13 @@ jobs: working-directory: packages/mcp-server run: pnpm pack --pack-destination dist - - name: Package telemetry - working-directory: packages/telemetry + - name: Package Spotlight SDK + working-directory: packages/spotlight run: pnpm pack --pack-destination dist - - name: Check packed telemetry consumer - working-directory: packages/telemetry - run: node scripts/check-package.mjs dist/sentry-telemetry-*.tgz + - name: Check packed Spotlight SDK consumer + working-directory: packages/spotlight + run: node scripts/check-package.mjs dist/sentry-spotlight-*.tgz - name: Archive artifacts uses: actions/upload-artifact@v4 @@ -53,11 +53,11 @@ jobs: path: | ${{ github.workspace }}/packages/mcp-server/dist/*.tgz - - name: Archive telemetry artifact + - name: Archive Spotlight SDK artifact uses: actions/upload-artifact@v4 with: - name: sentry-telemetry-${{ github.sha }} - path: ${{ github.workspace }}/packages/telemetry/dist/*.tgz + name: sentry-spotlight-${{ github.sha }} + path: ${{ github.workspace }}/packages/spotlight/dist/*.tgz if-no-files-found: error - name: Publish Test Report diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d9b48f0f2..293df4d37 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ on: options: - cli - mcp - - telemetry + - spotlight bump: description: Version bump type required: true @@ -69,9 +69,9 @@ jobs: workspace=packages/mcp-server package_path=packages/mcp-server ;; - telemetry) - workspace=packages/telemetry - package_path=packages/telemetry + spotlight) + workspace=packages/spotlight + package_path=packages/spotlight ;; *) echo "::error::Unsupported product: $PRODUCT" diff --git a/apps/local/README.md b/apps/local/README.md index a9f456b99..f8be6f4c7 100644 --- a/apps/local/README.md +++ b/apps/local/README.md @@ -13,7 +13,7 @@ The receiver can also be a named Local session or an explicitly selected remote receiver. Capture lifecycle and retention are managed through the existing Local controls. -The browser uses `@sentry/telemetry` for trace assembly, database and resource +The browser uses `@sentry/spotlight` for trace assembly, database and resource summaries, AI field interpretation, profile call trees and Web Vitals analysis. Raw envelopes, item headers and attachment payloads remain available for inspection. Each receiver connection owns a bounded, memory-only processing store; clearing diff --git a/apps/local/package.json b/apps/local/package.json index dbfb5e9c3..bd8a12276 100644 --- a/apps/local/package.json +++ b/apps/local/package.json @@ -4,20 +4,20 @@ "version": "0.0.0", "type": "module", "scripts": { - "dev": "pnpm --filter @sentry/telemetry run build && vite", - "build": "pnpm --filter @sentry/telemetry run build && tsc -b && vite build", + "dev": "pnpm --filter @sentry/spotlight run build && vite", + "build": "pnpm --filter @sentry/spotlight run build && tsc -b && vite build", "lint": "oxlint", "preview": "vite preview", - "test": "pnpm --filter @sentry/telemetry run build && vitest run", + "test": "pnpm --filter @sentry/spotlight run build && vitest run", "test:integration": "vitest run src/App.integration.test.tsx", - "typecheck": "pnpm --filter @sentry/telemetry run build && tsc -b", + "typecheck": "pnpm --filter @sentry/spotlight run build && tsc -b", "sample": "tsx scripts/replay.ts" }, "engines": { "node": "^20.19.0 || >=22.12.0" }, "dependencies": { - "@sentry/telemetry": "workspace:*", + "@sentry/spotlight": "workspace:*", "@hookform/resolvers": "^5.9.1", "@radix-ui/react-dialog": "^1.1.23", "@radix-ui/react-popover": "^1.3.1", diff --git a/apps/local/src/lib/inspection.ts b/apps/local/src/lib/inspection.ts index f59f3fa44..cf7eaecd2 100644 --- a/apps/local/src/lib/inspection.ts +++ b/apps/local/src/lib/inspection.ts @@ -1,8 +1,8 @@ -import { inspectAI, isAIInteraction as isAiPayload } from "@sentry/telemetry"; +import { inspectAI, isAIInteraction as isAiPayload } from "@sentry/spotlight"; export { isAIInteraction as isAiPayload, readSpanData as spanData, -} from "@sentry/telemetry"; +} from "@sentry/spotlight"; import { getEventMetadata, type LocalFeedItem } from "./spotlight.ts"; import { getEventExceptions, formatExceptionTitle } from "./exceptions.ts"; diff --git a/apps/local/src/lib/performance.ts b/apps/local/src/lib/performance.ts index 09acf8a28..d13a3bf11 100644 --- a/apps/local/src/lib/performance.ts +++ b/apps/local/src/lib/performance.ts @@ -3,7 +3,7 @@ import { groupWebVitalPages, type VitalVisit as SharedVitalVisit, type TelemetrySnapshot, -} from "@sentry/telemetry"; +} from "@sentry/spotlight"; import type { LocalFeedItem } from "./spotlight.ts"; import type { TraceGroup, TraceSpan } from "./trace-model.ts"; import { record, text } from "./inspection.ts"; @@ -77,8 +77,8 @@ export function spanInsights( }); } -export { WEB_VITALS as VITALS } from "@sentry/telemetry"; -export type { VitalName, VitalMeasurement } from "@sentry/telemetry"; +export { WEB_VITALS as VITALS } from "@sentry/spotlight"; +export type { VitalName, VitalMeasurement } from "@sentry/spotlight"; export type VitalVisit = { item: LocalFeedItem; metrics: SharedVitalVisit["metrics"]; diff --git a/apps/local/src/lib/profiles.ts b/apps/local/src/lib/profiles.ts index cfee379a7..74298cac4 100644 --- a/apps/local/src/lib/profiles.ts +++ b/apps/local/src/lib/profiles.ts @@ -1,4 +1,4 @@ -import { buildProfileModel, type ParsedProfile } from "@sentry/telemetry"; +import { buildProfileModel, type ParsedProfile } from "@sentry/spotlight"; import type { LocalFeedItem } from "./spotlight.ts"; import type { TraceGroup } from "./trace-model.ts"; import { record, finite, text } from "./inspection.ts"; @@ -6,7 +6,7 @@ export type { ProfileNode, ProfileFunction, ParsedProfile, -} from "@sentry/telemetry"; +} from "@sentry/spotlight"; export function parseProfile( item: LocalFeedItem, diff --git a/apps/local/src/lib/telemetry-regressions.test.ts b/apps/local/src/lib/telemetry-regressions.test.ts index c38889c2c..9dde2d6ec 100644 --- a/apps/local/src/lib/telemetry-regressions.test.ts +++ b/apps/local/src/lib/telemetry-regressions.test.ts @@ -1,4 +1,6 @@ import { expect, test } from "vitest"; +import type { ProcessedSpan } from "@sentry/spotlight"; +import { adaptTraceGroups } from "./trace-model.ts"; import { decodeEnvelope } from "./spotlight.ts"; import { createLocalTelemetryStore } from "./telemetry-store.ts"; import { spanInsights } from "./performance.ts"; @@ -74,3 +76,294 @@ test("production span-v2 resource sizes use known wire aliases and omit unknown spanInsights(store.getSnapshot().traceGroups, "resources"), ).toMatchObject([{ count: 3, averageBytes: 300, averageMs: 1000 }]); }); + +test.each(["queries", "resources"] as const)( + "production unnamed and blank %s spans remain discoverable with operation labels", + async (kind) => { + const operation = kind === "queries" ? "db.query" : "resource.script"; + const store = await capture([ + [ + { type: "transaction" }, + { + transaction: "GET /unnamed", + start_timestamp: 1, + timestamp: 2, + contexts: { trace: { trace_id: "unnamed", span_id: "root" } }, + spans: [ + { + span_id: "missing", + op: operation, + start_timestamp: 1, + timestamp: 1.1, + }, + { + span_id: "blank", + op: operation, + description: " ", + start_timestamp: 1.1, + timestamp: 1.2, + }, + { + span_id: "named", + op: operation, + description: " SELECT 1 ", + start_timestamp: 1.2, + timestamp: 1.3, + }, + ], + }, + ], + ]); + const summaries = spanInsights(store.getSnapshot().traceGroups, kind); + expect(summaries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: operation, + count: 2, + totalMs: 200, + averageMs: 100, + executions: [ + expect.objectContaining({ + span: expect.objectContaining({ + id: "missing", + description: operation, + }), + }), + expect.objectContaining({ + span: expect.objectContaining({ + id: "blank", + description: " ", + }), + }), + ], + }), + expect.objectContaining({ + name: "SELECT 1", + count: 1, + executions: [ + expect.objectContaining({ + span: expect.objectContaining({ + id: "named", + description: " SELECT 1 ", + }), + }), + ], + }), + ]), + ); + expect( + store.getSnapshot().traceGroups[0]?.processed?.spans.get("missing") + ?.description, + ).toBeUndefined(); + }, +); + +test("production missing parents under a real root retain descendants in the waterfall tree", async () => { + const store = await capture([ + [ + { type: "transaction" }, + { + transaction: "GET /partial", + start_timestamp: 1, + timestamp: 2, + contexts: { trace: { trace_id: "partial", span_id: "root" } }, + spans: [ + { + span_id: "child", + parent_span_id: "missing", + description: "child", + start_timestamp: 1.1, + timestamp: 1.2, + }, + { + span_id: "grandchild", + parent_span_id: "child", + description: "grandchild", + start_timestamp: 1.11, + timestamp: 1.15, + }, + ], + }, + ], + ]); + expect(store.getSnapshot().traceGroups[0]?.roots).toMatchObject([ + { + id: "root", + children: [ + { id: "child", parentId: "missing", children: [{ id: "grandchild" }] }, + ], + }, + ]); + expect(store.getSnapshot().traceGroups[0]?.spans).toHaveLength(3); +}); + +test("retention removes evicted spans from synchronous snapshots and every subscriber publication", async () => { + const store = createLocalTelemetryStore({ limit: 2 }); + function recordSpan(id: string, payload: unknown) { + const text = JSON.stringify([ + {}, + [[{ type: "span" }, { version: 2, items: [payload] }]], + ]); + store.recordEnvelope( + { id, type: "envelope", text, payload: text }, + decodeEnvelope(text, id), + ); + } + recordSpan("child-envelope", { + trace_id: "retained", + span_id: "evicted-child", + parent_span_id: "root", + start_timestamp: 1.1, + end_timestamp: 1.2, + }); + recordSpan("parent-envelope", { + trace_id: "retained", + span_id: "root", + name: "retained root", + is_segment: true, + start_timestamp: 1, + end_timestamp: 2, + }); + await store.drain(); + expect( + store.getSnapshot().traceGroups[0]?.spans.map((span) => span.id), + ).toContain("evicted-child"); + const publishedSpanIds: string[][] = []; + store.subscribe(() => + publishedSpanIds.push( + store + .getSnapshot() + .traceGroups.flatMap((trace) => trace.spans.map((span) => span.id)), + ), + ); + recordSpan("next-envelope", { + trace_id: "new", + span_id: "next", + start_timestamp: 2, + end_timestamp: 3, + }); + expect( + store + .getSnapshot() + .traceGroups.flatMap((trace) => trace.spans.map((span) => span.id)), + ).not.toContain("evicted-child"); + await store.drain(); + expect(publishedSpanIds.length).toBeGreaterThanOrEqual(2); + for (const ids of publishedSpanIds) + expect(ids).not.toContain("evicted-child"); + expect( + store + .getSnapshot() + .traceGroups.flatMap((trace) => trace.spans.map((span) => span.id)) + .sort(), + ).toEqual(["next", "root"]); +}); + +test("retention changes during shared ingestion cannot republish the previous window", async () => { + const store = createLocalTelemetryStore({ limit: 2 }); + function captureSpan(id: string) { + const text = JSON.stringify([ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "same-trace", + span_id: id, + start_timestamp: 1, + end_timestamp: 2, + }, + ], + }, + ], + ], + ]); + store.recordEnvelope( + { id, type: "envelope", text, payload: text }, + decodeEnvelope(text, id), + ); + } + captureSpan("evicted"); + await store.drain(); + captureSpan("second"); + // Let the shared ingestion start, then change retention before its await resumes. + await Promise.resolve(); + const published: string[][] = []; + store.subscribe(() => + published.push( + store + .getSnapshot() + .traceGroups.flatMap((trace) => trace.spans.map((span) => span.id)), + ), + ); + captureSpan("third"); + await store.drain(); + for (const ids of published) expect(ids).not.toContain("evicted"); + expect( + store + .getSnapshot() + .traceGroups[0]?.spans.map((span) => span.id) + .sort(), + ).toEqual(["second", "third"]); +}); + +test("nested orphan wrappers promote each real child once even when wrappers cycle", async () => { + const store = await capture([ + [ + { type: "transaction" }, + { + transaction: "partial tree", + start_timestamp: 1, + timestamp: 2, + contexts: { trace: { trace_id: "partial", span_id: "root" } }, + spans: [ + { + span_id: "child", + parent_span_id: "missing", + start_timestamp: 1.1, + timestamp: 1.2, + }, + ], + }, + ], + ]); + const snapshot = store.getSnapshot(); + const processed = snapshot.traceGroups[0]!.processed!; + const child = processed.spans.get("child")!; + const wrapper = { + ...child, + span_id: "missing", + op: "orphan", + children: [] as ProcessedSpan[], + }; + const nested: ProcessedSpan = { + ...wrapper, + span_id: "nested", + children: [wrapper, child], + }; + wrapper.children = [nested]; + const root: ProcessedSpan = { + ...processed.spans.get("root")!, + children: [wrapper], + }; + const [trace] = adaptTraceGroups(snapshot.items, [ + { + ...processed, + spans: new Map([ + ["root", root], + ["missing", wrapper], + ["nested", nested], + ["child", child], + ]), + spanTree: [root], + }, + ]); + expect(trace?.roots).toMatchObject([ + { id: "root", children: [{ id: "child" }] }, + ]); + expect(trace?.roots[0]?.children).toHaveLength(1); + expect(trace?.spans.map((span) => span.id)).toEqual(["root", "child"]); +}); diff --git a/apps/local/src/lib/telemetry-store.ts b/apps/local/src/lib/telemetry-store.ts index 81641af97..f6cf8f69a 100644 --- a/apps/local/src/lib/telemetry-store.ts +++ b/apps/local/src/lib/telemetry-store.ts @@ -1,4 +1,4 @@ -import { createTelemetryStore } from "@sentry/telemetry"; +import { createTelemetryStore } from "@sentry/spotlight"; import { adaptTraceGroups, type TraceGroup } from "./trace-model.ts"; import { aiItems } from "./inspection.ts"; import { @@ -148,6 +148,9 @@ export function createLocalTelemetryStore({ processedRawIds.add(raw.id); } if (epoch !== generation) return; + // A capture can change while ingestion yields. Rebuild against the + // latest retention window before publishing any of that work. + if (dirty) continue; publish( snapshot.items, snapshot.envelopes, @@ -188,15 +191,21 @@ export function createLocalTelemetryStore({ envelopeItems.set(envelope.id, decoded); const envelopes = [...snapshot.envelopes, envelope].slice(-limit); const retainedIds = new Set(envelopes.map((item) => item.id)); - for (const id of envelopeItems.keys()) - if (!retainedIds.has(id)) envelopeItems.delete(id); + for (const id of envelopeItems.keys()) { + if (!retainedIds.has(id)) { + envelopeItems.delete(id); + needsRebuild = true; + } + } + const items = envelopes.flatMap((item) => [ + ...(envelopeItems.get(item.id) ?? []), + ]); publish( - envelopes.flatMap((item) => [...(envelopeItems.get(item.id) ?? [])]), + items, envelopes, - adaptTraceGroups( - envelopes.flatMap((item) => [...(envelopeItems.get(item.id) ?? [])]), - processor.getTraces(), - ), + // The shared processor still contains the previous retention window + // until its asynchronous rebuild. Never expose those evicted spans. + adaptTraceGroups(items, needsRebuild ? [] : processor.getTraces()), ); updateProcessed(); }, diff --git a/apps/local/src/lib/trace-model.ts b/apps/local/src/lib/trace-model.ts index c1e646167..498c8445a 100644 --- a/apps/local/src/lib/trace-model.ts +++ b/apps/local/src/lib/trace-model.ts @@ -1,4 +1,4 @@ -import { type ProcessedTrace, type ProcessedSpan } from "@sentry/telemetry"; +import { type ProcessedTrace, type ProcessedSpan } from "@sentry/spotlight"; import { getEventMetadata, type EventMetadata, @@ -107,29 +107,34 @@ export function adaptTraceGroups( // use the JavaScript call stack merely to create presentation objects. const sharedSpans = [...trace.spans.values()]; for (const span of sharedSpans) if (span.op !== "orphan") adapt(span); + function visibleChildren(spans: readonly ProcessedSpan[]): TraceSpan[] { + const visible: TraceSpan[] = []; + const pending = [...spans].reverse(); + const visited = new Set(); + while (pending.length) { + const span = pending.pop()!; + if (visited.has(span.span_id)) continue; + visited.add(span.span_id); + if (span.op === "orphan") { + // Missing-parent wrappers can occur below real spans as well as at + // the root. Promote their captured children at every depth. + for ( + let index = (span.children?.length ?? 0) - 1; + index >= 0; + index-- + ) + pending.push(span.children![index]!); + } else { + visible.push(adapt(span)); + } + } + return visible; + } for (const span of sharedSpans) { const local = mapped.get(span.span_id); - if (local) - local.children = (span.children ?? []).flatMap((child) => { - const converted = mapped.get(child.span_id); - return converted ? [converted] : []; - }); - } - const roots: TraceSpan[] = []; - const pending = [...trace.spanTree].reverse(); - while (pending.length) { - const span = pending.pop()!; - if (span.op === "orphan") { - for ( - let index = (span.children?.length ?? 0) - 1; - index >= 0; - index-- - ) - pending.push(span.children![index]!); - } else { - roots.push(adapt(span)); - } + if (local) local.children = visibleChildren(span.children ?? []); } + const roots = visibleChildren(trace.spanTree); return [ { id: trace.trace_id, diff --git a/apps/local/src/test-fixtures/trace-groups.ts b/apps/local/src/test-fixtures/trace-groups.ts index c257055a1..2f532ef74 100644 --- a/apps/local/src/test-fixtures/trace-groups.ts +++ b/apps/local/src/test-fixtures/trace-groups.ts @@ -1,4 +1,4 @@ -import { createTelemetryStore } from "@sentry/telemetry"; +import { createTelemetryStore } from "@sentry/spotlight"; import { adaptTraceGroups, traceIdForItem, diff --git a/docs/README.md b/docs/README.md index 954f04278..56aa7f9ef 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,7 +16,7 @@ and package it from that workspace. The contributor index here links to its source and to . MCP documentation lives in the topic directories below; the MCP index explains which guides apply to that product. -Shared telemetry processing for the receiver and browser UI lives in [`@sentry/telemetry`](../packages/telemetry/README.md). +Shared telemetry processing for the receiver and browser UI lives in [`@sentry/spotlight`](../packages/spotlight/README.md). ## Shared Contributor Guides diff --git a/docs/cli/local-control-plane.md b/docs/cli/local-control-plane.md index 24fb831e8..d9a19094a 100644 --- a/docs/cli/local-control-plane.md +++ b/docs/cli/local-control-plane.md @@ -77,7 +77,7 @@ Increasing the limit preserves existing envelopes, their IDs, and active streams previously evicted envelopes cannot be recovered. Lowering below the retained count is rejected without changing any data. Clear envelopes first to reduce that limit. Changing capacity does not reset the session's three-hour lifecycle retention. -The receiver decodes and processes envelopes with `@sentry/telemetry/node`, shared +The receiver decodes and processes envelopes with `@sentry/spotlight/node`, shared with the browser viewer's telemetry engine. Envelope retention also has a 64 MiB parsed-data budget. Raw downloads have a separate 64 MiB decoded-body budget; a raw download returns 404 once its decoded bytes have been evicted. Both budgets diff --git a/packages/cli/package.json b/packages/cli/package.json index 5221fd07a..0bab59c5e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -61,7 +61,7 @@ "build": "pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/build.ts --single", "build:all": "pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/build.ts", "bundle": "pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:schema && pnpm run generate:docs && pnpm run generate:sdk && pnpm tsx script/bundle.ts", - "typecheck": "pnpm --filter @sentry/telemetry run build && pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:docs && pnpm run generate:sdk && tsc --noEmit", + "typecheck": "pnpm --filter @sentry/spotlight run build && pnpm --filter @sentry/mcp-core run build && pnpm --filter @sentry/mcp-server run build && pnpm run generate:docs && pnpm run generate:sdk && tsc --noEmit", "lint": "pnpm -w exec oxlint --report-unused-disable-directives packages/cli && pnpm -w exec oxfmt --check packages/cli", "lint:fix": "pnpm -w exec oxlint --fix packages/cli && pnpm -w exec oxfmt --write packages/cli", "test": "pnpm run test:unit", @@ -76,7 +76,7 @@ "generate:grammar": "node ../../scripts/generate-parsers.mjs .", "generate:completions": "pnpm tsx script/generate-completion-targets.ts", "generate:docs-sections": "pnpm tsx script/generate-docs-sections.ts", - "generate:schema": "pnpm --filter @sentry/telemetry run build && pnpm tsx script/generate-api-schema.ts", + "generate:schema": "pnpm --filter @sentry/spotlight run build && pnpm tsx script/generate-api-schema.ts", "generate:command-docs": "pnpm tsx script/generate-command-docs.ts", "eval:skill": "pnpm tsx script/eval-skill.ts", "bench": "pnpm tsx script/bench.ts", @@ -108,7 +108,7 @@ "@sentry/sqlish": "^1.0.1", "@sentry/symbolic": "13.7.0", "@sentry/toolkit-core": "workspace:*", - "@sentry/telemetry": "workspace:*", + "@sentry/spotlight": "workspace:*", "@stricli/auto-complete": "^1.2.8", "@stricli/core": "1.2.8", "@types/http-cache-semantics": "^4.2.0", diff --git a/packages/cli/src/commands/local/telemetry-buffer.ts b/packages/cli/src/commands/local/telemetry-buffer.ts index 022d28d71..a49acc7b7 100644 --- a/packages/cli/src/commands/local/telemetry-buffer.ts +++ b/packages/cli/src/commands/local/telemetry-buffer.ts @@ -6,8 +6,8 @@ import { createTelemetryPipeline, type TelemetryPayload, type TelemetryPipeline, -} from "@sentry/telemetry/node"; -import type { IgnoreReason, IngestResult } from "@sentry/telemetry"; +} from "@sentry/spotlight/node"; +import type { IgnoreReason, IngestResult } from "@sentry/spotlight"; import { MAX_ENVELOPES } from "./retention-limits.js"; /** Parsed wire records consumed by the existing terminal and SSE formatters. */ diff --git a/packages/telemetry/LICENSE b/packages/spotlight/LICENSE similarity index 100% rename from packages/telemetry/LICENSE rename to packages/spotlight/LICENSE diff --git a/packages/telemetry/README.md b/packages/spotlight/README.md similarity index 94% rename from packages/telemetry/README.md rename to packages/spotlight/README.md index 183b75f88..81d9d5606 100644 --- a/packages/telemetry/README.md +++ b/packages/spotlight/README.md @@ -1,4 +1,4 @@ -# `@sentry/telemetry` +# `@sentry/spotlight` This package shares Sentry envelope normalization, trace assembly, and insight calculations across server and frontend consumers. Each instance owns its state; feed server and @@ -9,15 +9,15 @@ comparing their retained histories. | Entry | Factory | Input | | ------------------------ | --------------------------- | ------------------------------------------------------------------ | -| `@sentry/telemetry` | `createTelemetryStore()` | Parsed Sentry envelopes in portable runtimes | -| `@sentry/telemetry/node` | `createTelemetryPipeline()` | Adds serialized bodies, decompression, and decoded payload history | +| `@sentry/spotlight` | `createTelemetryStore()` | Parsed Sentry envelopes in portable runtimes | +| `@sentry/spotlight/node` | `createTelemetryPipeline()` | Adds serialized bodies, decompression, and decoded payload history | ## Ingest and read telemetry -Use `@sentry/telemetry` for parsed Sentry envelopes in a browser, worker, or Node application. Use `@sentry/telemetry/node` for serialized or compressed payloads on Node 20+. Both expose the same processing and read APIs; creating an instance does not start a server. The portable entry has no Node or React dependency. +Use `@sentry/spotlight` for parsed Sentry envelopes in a browser, worker, or Node application. Use `@sentry/spotlight/node` for serialized or compressed payloads on Node 20+. Both expose the same processing and read APIs; creating an instance does not start a server. The portable entry has no Node or React dependency. ```ts -import { createTelemetryStore, serializeSnapshot } from "@sentry/telemetry"; +import { createTelemetryStore, serializeSnapshot } from "@sentry/spotlight"; const telemetry = createTelemetryStore(); const receipt = await telemetry.ingestEnvelope(parsedEnvelope, { @@ -39,7 +39,7 @@ telemetry.dispose(); The Node adapter accepts `string`, `Uint8Array`, and `Buffer` bodies. `ingestPayload({ body, contentType?, encoding?, userAgent?, deliveryId?, signal? })` decodes `application/x-sentry-envelope` (the default content type) and supports identity, gzip, deflate, and Brotli encoding. Use `getDecodedPayload(envelopeId)` for a copied decompressed payload when it remains in the independently bounded history. Parsed envelopes still use `ingestEnvelope`; the payload methods are `ingestPayload` and `ingestPayloads`. ```ts -import { createTelemetryPipeline } from "@sentry/telemetry/node"; +import { createTelemetryPipeline } from "@sentry/spotlight/node"; const pipeline = createTelemetryPipeline({ decodedHistory: { maxEntries: 100 }, @@ -132,23 +132,23 @@ Limits must be positive safe integers; Node `decodedHistory.maxEntries` and `dec `SDK_CAPABILITIES` exposes readonly machine-readable metadata for legacy and v2 spans, profile v1/v2, supported item types, and timestamp units. Supported item types include event/error, transaction, span, log, profile/profile_chunk, feedback, and attachment; individual wire versions can still be unsupported. Portable runtimes need standard JavaScript APIs including `AbortController`, `TextEncoder`, timers, maps, and sets. React is required only by your React consumer; compressed payload decoding is provided by the Node entry. -The [SDK examples](examples) cover a bounded HTTP server, a frontend transport adapter, a reader-only consumer, React subscription, and an executable lifecycle walkthrough with receipts, retry, partial batches, cancellation, drain, JSON, and paging. Run `pnpm --filter @sentry/telemetry test` to typecheck the published declarations and every example, then execute the lifecycle walkthrough. CI runs this check after the package build; it does not add a whole-project TypeScript gate. +The [SDK examples](examples) cover a bounded HTTP server, a frontend transport adapter, a reader-only consumer, React subscription, and an executable lifecycle walkthrough with receipts, retry, partial batches, cancellation, drain, JSON, and paging. Run `pnpm --filter @sentry/spotlight test` to typecheck the published declarations and every example, then execute the lifecycle walkthrough. CI runs this check after the package build; it does not add a whole-project TypeScript gate. ### Getter reference -| Read | Result | -| --------------------------------------------------------------- | --------------------------------------------------------------- | -| `getTrace(traceId)`, `getTraces(filter?)` | Assembled traces | -| `getEvent(eventId)`, `getEvents(filter?)`, `getErrors(filter?)` | Normalized events and captured exceptions | -| `getSpans(filter?)` | Normalized spans | -| `getLog(logId)`, `getLogs(filter?)` | Logs and trace correlation | -| `getDatabaseQuerySummaries(filter?)` | Database spans grouped by trimmed description | -| `getResourceSummaries(filter?)` | Resource timings and encoded sizes | -| `getProfile(traceId)`, `getProfileFunctionSummaries(filter?)` | Associated profiles and inclusive frame durations/sample counts | -| `getAITraces(filter?)` | AI interactions, tokens, prompts, responses, and tools | -| `getWebVitals(filter?)` | Scored page-load events | -| `getEnvelope(envelopeId)`, `getEnvelopes()` | Retained wire envelopes | -| `getSdks()` | Observed SDK identities | +| Read | Result | +| --------------------------------------------------------------- | -------------------------------------------------------------------- | +| `getTrace(traceId)`, `getTraces(filter?)` | Assembled traces | +| `getEvent(eventId)`, `getEvents(filter?)`, `getErrors(filter?)` | Normalized events and captured exceptions | +| `getSpans(filter?)` | Normalized spans | +| `getLog(logId)`, `getLogs(filter?)` | Logs and trace correlation | +| `getDatabaseQuerySummaries(filter?)` | Database spans grouped by trimmed description, operation, or span ID | +| `getResourceSummaries(filter?)` | Resource timings and encoded sizes | +| `getProfile(traceId)`, `getProfileFunctionSummaries(filter?)` | Associated profiles and inclusive frame durations/sample counts | +| `getAITraces(filter?)` | AI interactions, tokens, prompts, responses, and tools | +| `getWebVitals(filter?)` | Scored page-load events | +| `getEnvelope(envelopeId)`, `getEnvelopes()` | Retained wire envelopes | +| `getSdks()` | Observed SDK identities | Event reads accept `traceId` and `spanId`; trace/log/span/profile/summary reads accept `traceId`; query summaries also accept `description`. The time/order/limit @@ -177,7 +177,7 @@ import { buildProfileModel, groupWebVitalPages, inspectAI, -} from "@sentry/telemetry"; +} from "@sentry/spotlight"; const profile = buildProfileModel(rawProfilePayload, selectedThreadId); // profile.root, profile.functions, profile.threads; durations are milliseconds. @@ -194,12 +194,12 @@ const pages = groupWebVitalPages([ ## Ownership and adoption -This Apache-2.0 implementation originated in [Spotlight](https://github.com/getsentry/spotlight/pull/1374). Toolkit owns subsequent shared processing changes. The Local receiver imports `@sentry/telemetry/node`; the Local browser app imports `@sentry/telemetry`. UI rendering, HTTP/SSE transport, sessions, and terminal formatting stay in their applications. +This Apache-2.0 implementation originated in [Spotlight](https://github.com/getsentry/spotlight/pull/1374). Toolkit owns subsequent shared processing changes. The Local receiver imports `@sentry/spotlight/node`; the Local browser app imports `@sentry/spotlight`. UI rendering, HTTP/SSE transport, sessions, and terminal formatting stay in their applications. -The package exports built JavaScript and declarations. Run `pnpm --filter @sentry/telemetry build` before direct source development in a consumer. Workspace build/test scripts prepare this dependency. The portable entry has no React or Node runtime dependency. +The package exports built JavaScript and declarations. Run `pnpm --filter @sentry/spotlight build` before direct source development in a consumer. Workspace build/test scripts prepare this dependency. The portable entry has no React or Node runtime dependency. After the first npm release, Spotlight can replace its extraction with imports from this package and remove its local processing implementation. Existing Spotlight `/sdk` transport helpers are a separate compatibility API. No npm publication occurs as part of the migration PR. ## Release -Use the repository Release workflow with product `telemetry`. The `packages/telemetry` Craft workspace publishes the independently versioned `@sentry/telemetry` npm artifact built by Post-merge tasks. The portable and Node consumer contracts and executable examples are checked against emitted declarations before packing. +Use the repository Release workflow with product `spotlight`. The `packages/spotlight` Craft workspace publishes the independently versioned `@sentry/spotlight` npm artifact built by Post-merge tasks. The portable and Node consumer contracts and executable examples are checked against emitted declarations before packing. diff --git a/packages/telemetry/_fixtures/continuous_profiling/profile_chunk_1.txt b/packages/spotlight/_fixtures/continuous_profiling/profile_chunk_1.txt similarity index 100% rename from packages/telemetry/_fixtures/continuous_profiling/profile_chunk_1.txt rename to packages/spotlight/_fixtures/continuous_profiling/profile_chunk_1.txt diff --git a/packages/telemetry/_fixtures/continuous_profiling/transaction.txt b/packages/spotlight/_fixtures/continuous_profiling/transaction.txt similarity index 100% rename from packages/telemetry/_fixtures/continuous_profiling/transaction.txt rename to packages/spotlight/_fixtures/continuous_profiling/transaction.txt diff --git a/packages/telemetry/_fixtures/envelope_span_v2.txt b/packages/spotlight/_fixtures/envelope_span_v2.txt similarity index 100% rename from packages/telemetry/_fixtures/envelope_span_v2.txt rename to packages/spotlight/_fixtures/envelope_span_v2.txt diff --git a/packages/telemetry/examples/frontend.ts b/packages/spotlight/examples/frontend.ts similarity index 97% rename from packages/telemetry/examples/frontend.ts rename to packages/spotlight/examples/frontend.ts index 7fa946801..0cba4b928 100644 --- a/packages/telemetry/examples/frontend.ts +++ b/packages/spotlight/examples/frontend.ts @@ -1,4 +1,4 @@ -import { type IngestOptions, createTelemetryStore } from "@sentry/telemetry"; +import { type IngestOptions, createTelemetryStore } from "@sentry/spotlight"; /** Keep one instance for the lifetime of the telemetry view. */ export function createFrontendTelemetry() { diff --git a/packages/telemetry/examples/lifecycle.ts b/packages/spotlight/examples/lifecycle.ts similarity index 96% rename from packages/telemetry/examples/lifecycle.ts rename to packages/spotlight/examples/lifecycle.ts index a00914ae6..4381798ec 100644 --- a/packages/telemetry/examples/lifecycle.ts +++ b/packages/spotlight/examples/lifecycle.ts @@ -1,5 +1,5 @@ -import { SDK_CAPABILITIES, type SentryEnvelope } from "@sentry/telemetry"; -import { createTelemetryPipeline } from "@sentry/telemetry/node"; +import { SDK_CAPABILITIES, type SentryEnvelope } from "@sentry/spotlight"; +import { createTelemetryPipeline } from "@sentry/spotlight/node"; import { createFrontendTelemetry } from "./frontend.js"; import { inspectTrace, readEventIds } from "./reader.js"; diff --git a/packages/telemetry/examples/react.tsx b/packages/spotlight/examples/react.tsx similarity index 91% rename from packages/telemetry/examples/react.tsx rename to packages/spotlight/examples/react.tsx index 31db31e92..774548d9d 100644 --- a/packages/telemetry/examples/react.tsx +++ b/packages/spotlight/examples/react.tsx @@ -1,4 +1,4 @@ -import type { TelemetryReader, TelemetryStore } from "@sentry/telemetry"; +import type { TelemetryReader, TelemetryStore } from "@sentry/spotlight"; import { useSyncExternalStore } from "react"; type Props = { diff --git a/packages/telemetry/examples/reader.ts b/packages/spotlight/examples/reader.ts similarity index 97% rename from packages/telemetry/examples/reader.ts rename to packages/spotlight/examples/reader.ts index 507d60c75..780a9fbd3 100644 --- a/packages/telemetry/examples/reader.ts +++ b/packages/spotlight/examples/reader.ts @@ -3,7 +3,7 @@ import { serializeSnapshot, serializeSummaries, serializeTrace, -} from "@sentry/telemetry"; +} from "@sentry/spotlight"; /** Business logic only needs a reader, independent of how telemetry arrives. */ export function inspectTrace(reader: TelemetryReader, traceId: string) { diff --git a/packages/telemetry/examples/server.ts b/packages/spotlight/examples/server.ts similarity index 97% rename from packages/telemetry/examples/server.ts rename to packages/spotlight/examples/server.ts index 0c0063402..b3a0d2529 100644 --- a/packages/telemetry/examples/server.ts +++ b/packages/spotlight/examples/server.ts @@ -1,5 +1,5 @@ import { createServer } from "node:http"; -import { createTelemetryPipeline } from "@sentry/telemetry/node"; +import { createTelemetryPipeline } from "@sentry/spotlight/node"; const maxPayloadBytes = 8 * 1024 * 1024; diff --git a/packages/telemetry/examples/tsconfig.json b/packages/spotlight/examples/tsconfig.json similarity index 100% rename from packages/telemetry/examples/tsconfig.json rename to packages/spotlight/examples/tsconfig.json diff --git a/packages/telemetry/package.json b/packages/spotlight/package.json similarity index 95% rename from packages/telemetry/package.json rename to packages/spotlight/package.json index ead247d89..7a36ef754 100644 --- a/packages/telemetry/package.json +++ b/packages/spotlight/package.json @@ -1,5 +1,5 @@ { - "name": "@sentry/telemetry", + "name": "@sentry/spotlight", "version": "0.1.0", "description": "Shared Sentry envelope processing, trace assembly, and telemetry queries for browsers and servers", "type": "module", @@ -7,7 +7,7 @@ "repository": { "type": "git", "url": "https://github.com/getsentry/toolkit.git", - "directory": "packages/telemetry" + "directory": "packages/spotlight" }, "engines": { "node": ">=20.19.0" diff --git a/packages/telemetry/scripts/check-package.mjs b/packages/spotlight/scripts/check-package.mjs similarity index 96% rename from packages/telemetry/scripts/check-package.mjs rename to packages/spotlight/scripts/check-package.mjs index 915c4678f..17ab4c270 100644 --- a/packages/telemetry/scripts/check-package.mjs +++ b/packages/spotlight/scripts/check-package.mjs @@ -82,7 +82,7 @@ try { ); const scope = join(consumerDirectory, "node_modules", "@sentry"); await mkdir(scope, { recursive: true }); - await symlink(packedDirectory, join(scope, "telemetry"), "junction"); + await symlink(packedDirectory, join(scope, "spotlight"), "junction"); const manifest = JSON.parse( await readFile(join(packedDirectory, "package.json"), "utf8"), ); @@ -123,7 +123,7 @@ try { ); } catch { throw new Error( - "Build @sentry/telemetry before checking its public package", + "Build @sentry/spotlight before checking its public package", ); } } @@ -142,7 +142,7 @@ try { if (!tarball) { const scope = join(output, "node_modules", "@sentry"); await mkdir(scope, { recursive: true }); - await symlink(packageDirectory, join(scope, "telemetry"), "junction"); + await symlink(packageDirectory, join(scope, "spotlight"), "junction"); } run(join(output, "lifecycle.js")); console.log( diff --git a/packages/telemetry/src/sdk/core.ts b/packages/spotlight/src/sdk/core.ts similarity index 100% rename from packages/telemetry/src/sdk/core.ts rename to packages/spotlight/src/sdk/core.ts diff --git a/packages/telemetry/src/sdk/node.spec.ts b/packages/spotlight/src/sdk/node.spec.ts similarity index 100% rename from packages/telemetry/src/sdk/node.spec.ts rename to packages/spotlight/src/sdk/node.spec.ts diff --git a/packages/telemetry/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts similarity index 100% rename from packages/telemetry/src/sdk/node.ts rename to packages/spotlight/src/sdk/node.ts diff --git a/packages/spotlight/src/telemetry/ai-regressions.spec.ts b/packages/spotlight/src/telemetry/ai-regressions.spec.ts new file mode 100644 index 000000000..5fc0b6f31 --- /dev/null +++ b/packages/spotlight/src/telemetry/ai-regressions.spec.ts @@ -0,0 +1,229 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "../sdk/core.ts"; + +function span( + id: string, + data: Record = {}, + parent = "transaction", + op = "gen_ai.chat", +) { + return { + trace_id: "ai-trace", + span_id: id, + parent_span_id: parent, + op, + start_timestamp: 1, + timestamp: 2, + data, + }; +} + +async function ingestSpans( + store: ReturnType, + spans: ReturnType[], +) { + const result = await store.ingestEnvelope([ + {}, + [ + [ + { type: "transaction" }, + { + type: "transaction", + event_id: "transaction-event", + transaction: "AI request", + start_timestamp: 1, + timestamp: 2, + contexts: { + trace: { + trace_id: "ai-trace", + span_id: "transaction", + op: "http.server", + }, + }, + spans, + }, + ], + ], + ]); + expect(result.status).toBe("processed"); +} + +const calls = [ + { toolCallId: "weather-call", toolName: "weather", args: { city: "Pune" } }, +]; + +test.each(["generic", "vercel"])( + "AI queries preserve descendant response order and shallowest %s roots", + async (provider) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span( + "root", + provider === "generic" + ? { "gen_ai.request.model": "root-model" } + : { "vercel.ai.model.id": "root-model" }, + ), + span("first", { "gen_ai.response.text": "first" }, "root"), + span("nested", { "gen_ai.response.text": "nested" }, "first"), + span("second", { "gen_ai.response.text": "second" }, "root"), + span( + "separate", + provider === "generic" + ? { "gen_ai.request.model": "separate-model" } + : { "vercel.ai.model.id": "separate-model" }, + ), + ]); + const traces = store.getAITraces(); + expect(traces.map((trace) => trace.id)).toEqual(["root", "separate"]); + expect(traces[0]?.response?.text).toBe("second"); + expect(traces[0]?.metadata.modelId).toBe("root-model"); + } finally { + store.dispose(); + } + }, +); + +test.each(["[{", "not JSON", "null", "{}", "3", '"text"', {}, 3, true])( + "AI queries retain other response data when Vercel tool calls are malformed: %j", + async (toolCalls) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.response.toolCalls": JSON.stringify(calls), + }), + span( + "malformed", + { + "vercel.ai.response.toolCalls": toolCalls, + "gen_ai.response.text": "Still available", + "vercel.ai.response.finishReason": "stop", + }, + "root", + ), + span("sibling", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.model.id": "sibling-model", + }), + ]); + const traces = store.getAITraces(); + expect(traces).toHaveLength(2); + expect(traces.find((trace) => trace.id === "root")?.response).toEqual({ + toolCalls: calls, + text: "Still available", + finishReason: "stop", + }); + expect( + traces.find((trace) => trace.id === "sibling")?.metadata.modelId, + ).toBe("sibling-model"); + } finally { + store.dispose(); + } + }, +); + +test("AI queries accept structured Vercel response tool-call arrays", async () => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "vercel.ai.response.toolCalls": calls, + }), + ]); + expect(store.getAITraces()[0]?.response?.toolCalls).toEqual(calls); + } finally { + store.dispose(); + } +}); + +test.each([0, "0"])( + "AI queries preserve explicit zero retry and step settings: %j", + async (zero) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "vercel.ai.settings.maxRetries": zero, + "vercel.ai.settings.maxSteps": zero, + }), + ]); + expect(store.getAITraces()[0]?.metadata).toMatchObject({ + maxRetries: 0, + maxSteps: 0, + }); + } finally { + store.dispose(); + } + }, +); + +test.each( + [null, "", " ", false, true, [], {}, "invalid", "Infinity"].map((value) => ({ + value, + })), +)( + "AI queries ignore malformed retry and step settings: $value", + async ({ value }) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "vercel.ai.settings.maxRetries": 2, + "vercel.ai.settings.maxSteps": "3", + }), + span( + "child", + { + "vercel.ai.settings.maxRetries": value, + "vercel.ai.settings.maxSteps": value, + }, + "root", + ), + ]); + expect(store.getAITraces()[0]?.metadata).toMatchObject({ + maxRetries: 2, + maxSteps: 3, + }); + } finally { + store.dispose(); + } + }, +); + +test.each([ + { provider: "generic", aiAtRoot: true }, + { provider: "generic", aiAtRoot: false }, + { provider: "vercel", aiAtRoot: true }, + { provider: "vercel", aiAtRoot: false }, +])( + "AI queries traverse deep $provider trees with aiAtRoot=$aiAtRoot", + async ({ provider, aiAtRoot }) => { + const store = createTelemetryStore(); + try { + const spans = Array.from({ length: 12000 }, (_, index) => { + const ai = aiAtRoot ? index === 0 : index === 11999; + return span( + `span-${index}`, + ai + ? provider === "generic" + ? { "gen_ai.request.model": "deep-model" } + : { "vercel.ai.model.id": "deep-model" } + : {}, + index === 0 ? "transaction" : `span-${index - 1}`, + ai ? "gen_ai.chat" : "function", + ); + }); + await ingestSpans(store, spans); + const traces = store.getAITraces(); + expect(traces).toHaveLength(1); + expect(traces[0]).toMatchObject({ + id: aiAtRoot ? "span-0" : "span-11999", + metadata: { modelId: "deep-model" }, + }); + } finally { + store.dispose(); + } + }, +); diff --git a/packages/telemetry/src/telemetry/ai/aiLibraries.ts b/packages/spotlight/src/telemetry/ai/aiLibraries.ts similarity index 100% rename from packages/telemetry/src/telemetry/ai/aiLibraries.ts rename to packages/spotlight/src/telemetry/ai/aiLibraries.ts diff --git a/packages/telemetry/src/telemetry/ai/genAI.ts b/packages/spotlight/src/telemetry/ai/genAI.ts similarity index 93% rename from packages/telemetry/src/telemetry/ai/genAI.ts rename to packages/spotlight/src/telemetry/ai/genAI.ts index 08e089748..d8bc8a63b 100644 --- a/packages/telemetry/src/telemetry/ai/genAI.ts +++ b/packages/spotlight/src/telemetry/ai/genAI.ts @@ -83,21 +83,30 @@ export const genAIHandler: AILibraryHandler = { extractRootSpans: (spans: Span[]): Span[] => { const resultRoots: Span[] = []; - - const findAndCollectAIRoots = (spansToSearch: Span[]) => { - for (const currentSpan of spansToSearch) { - if (isGenAIConventionSpan(currentSpan)) { - resultRoots.push(currentSpan); - } else if (!isGenAISpan(currentSpan) && currentSpan.children?.length) { - // Stop at any gen_ai.* span: it's the shallowest AI root for its - // tree (claimed by this handler or the Vercel one), so its nested - // gen_ai.* children must not be surfaced as separate roots. - findAndCollectAIRoots(currentSpan.children); + const pending = [...spans].reverse(); + const visited = new Set(); + for ( + let currentSpan = pending.pop(); + currentSpan; + currentSpan = pending.pop() + ) { + if (visited.has(currentSpan)) continue; + visited.add(currentSpan); + if (isGenAIConventionSpan(currentSpan)) { + resultRoots.push(currentSpan); + } else if (!isGenAISpan(currentSpan)) { + // Stop at any gen_ai.* span: it's the shallowest AI root for its + // tree (claimed by this handler or the Vercel one), so its nested + // gen_ai.* children must not be surfaced as separate roots. + for ( + let index = (currentSpan.children?.length ?? 0) - 1; + index >= 0; + index-- + ) { + pending.push(currentSpan.children![index]); } } - }; - - findAndCollectAIRoots(spans); + } return resultRoots; }, diff --git a/packages/telemetry/src/telemetry/ai/inspection.ts b/packages/spotlight/src/telemetry/ai/inspection.ts similarity index 100% rename from packages/telemetry/src/telemetry/ai/inspection.ts rename to packages/spotlight/src/telemetry/ai/inspection.ts diff --git a/packages/telemetry/src/telemetry/ai/vercelAISDK.ts b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts similarity index 91% rename from packages/telemetry/src/telemetry/ai/vercelAISDK.ts rename to packages/spotlight/src/telemetry/ai/vercelAISDK.ts index 3d20a8169..f38ffbf6a 100644 --- a/packages/telemetry/src/telemetry/ai/vercelAISDK.ts +++ b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts @@ -120,18 +120,27 @@ export const vercelAISDKHandler: AILibraryHandler = { extractRootSpans: (spans: Span[]): Span[] => { const resultRoots: Span[] = []; - - const findAndCollectAIRoots = (spansToSearch: Span[]) => { - for (const currentSpan of spansToSearch) { - if (currentSpan.op?.toLowerCase().startsWith(AI_SPAN_OP_PREFIX)) { - resultRoots.push(currentSpan); - } else if (currentSpan.children?.length) { - findAndCollectAIRoots(currentSpan.children); + const pending = [...spans].reverse(); + const visited = new Set(); + for ( + let currentSpan = pending.pop(); + currentSpan; + currentSpan = pending.pop() + ) { + if (visited.has(currentSpan)) continue; + visited.add(currentSpan); + if (currentSpan.op?.toLowerCase().startsWith(AI_SPAN_OP_PREFIX)) { + resultRoots.push(currentSpan); + } else { + for ( + let index = (currentSpan.children?.length ?? 0) - 1; + index >= 0; + index-- + ) { + pending.push(currentSpan.children![index]); } } - }; - - findAndCollectAIRoots(spans); + } return resultRoots; }, @@ -381,14 +390,14 @@ function extractAIMetadata(span: Span, trace: AITrace) { } } - if (span.data[AI_SETTINGS_MAX_RETRIES_FIELD]) { - trace.metadata.maxRetries = Number( - span.data[AI_SETTINGS_MAX_RETRIES_FIELD], - ); + const maxRetries = finiteSetting(span.data[AI_SETTINGS_MAX_RETRIES_FIELD]); + if (maxRetries !== undefined) { + trace.metadata.maxRetries = maxRetries; } - if (span.data[AI_SETTINGS_MAX_STEPS_FIELD]) { - trace.metadata.maxSteps = Number(span.data[AI_SETTINGS_MAX_STEPS_FIELD]); + const maxSteps = finiteSetting(span.data[AI_SETTINGS_MAX_STEPS_FIELD]); + if (maxSteps !== undefined) { + trace.metadata.maxSteps = maxSteps; } // Token usage - use === undefined to allow 0 values @@ -410,6 +419,17 @@ function extractAIMetadata(span: Span, trace: AITrace) { } } +function finiteSetting(value: unknown): number | undefined { + if ( + typeof value !== "number" && + (typeof value !== "string" || !value.trim()) + ) { + return undefined; + } + const number = Number(value); + return Number.isFinite(number) ? number : undefined; +} + function extractTelemetryMetadata(span: Span, trace: AITrace) { if (!span.data) return; @@ -493,7 +513,15 @@ function extractResponseData(span: Span, trace: AITrace) { const toolCalls = span.data[AI_RESPONSE_TOOL_CALLS_FIELD]; if (toolCalls) { - trace.response.toolCalls = JSON.parse(stringifyValue(toolCalls)); + try { + const parsed: unknown = + typeof toolCalls === "string" ? JSON.parse(toolCalls) : toolCalls; + if (Array.isArray(parsed)) { + trace.response.toolCalls = parsed; + } + } catch { + // Truncated tool-call JSON must not discard the rest of the AI trace. + } } } diff --git a/packages/telemetry/src/telemetry/attachments.spec.ts b/packages/spotlight/src/telemetry/attachments.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/attachments.spec.ts rename to packages/spotlight/src/telemetry/attachments.spec.ts diff --git a/packages/telemetry/src/telemetry/capabilities.ts b/packages/spotlight/src/telemetry/capabilities.ts similarity index 100% rename from packages/telemetry/src/telemetry/capabilities.ts rename to packages/spotlight/src/telemetry/capabilities.ts diff --git a/packages/telemetry/src/telemetry/client.spec.ts b/packages/spotlight/src/telemetry/client.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/client.spec.ts rename to packages/spotlight/src/telemetry/client.spec.ts diff --git a/packages/telemetry/src/telemetry/constants/performance.ts b/packages/spotlight/src/telemetry/constants/performance.ts similarity index 100% rename from packages/telemetry/src/telemetry/constants/performance.ts rename to packages/spotlight/src/telemetry/constants/performance.ts diff --git a/packages/telemetry/src/telemetry/constants/sentry.ts b/packages/spotlight/src/telemetry/constants/sentry.ts similarity index 100% rename from packages/telemetry/src/telemetry/constants/sentry.ts rename to packages/spotlight/src/telemetry/constants/sentry.ts diff --git a/packages/telemetry/src/telemetry/contracts.ts b/packages/spotlight/src/telemetry/contracts.ts similarity index 100% rename from packages/telemetry/src/telemetry/contracts.ts rename to packages/spotlight/src/telemetry/contracts.ts diff --git a/packages/spotlight/src/telemetry/data/profiles.test.ts b/packages/spotlight/src/telemetry/data/profiles.test.ts new file mode 100644 index 000000000..1434a6402 --- /dev/null +++ b/packages/spotlight/src/telemetry/data/profiles.test.ts @@ -0,0 +1,275 @@ +import { describe, expect, it } from "vitest"; +import type { SentryProfileWithTraceMeta, Span, Trace } from "../types.ts"; +import { getSpansFromProfile, graftProfileSpans } from "./profiles.ts"; + +function trace(): Trace { + return { + trace_id: "trace", + start_timestamp: 0, + timestamp: 10, + transactions: [], + errors: 0, + rootTransaction: null, + rootTransactionName: "test", + spans: new Map(), + spanTree: [], + profileGrafted: false, + }; +} + +function profile( + overrides: Partial = {}, +): SentryProfileWithTraceMeta { + return { + timestamp: 0, + active_thread_id: "main", + frames: [{ module: "app", function: "main" }], + stacks: [[0]], + samples: [{ thread_id: "main", stack_id: 0, start_timestamp: 0 }], + ...overrides, + }; +} + +describe("profile span frames", () => { + it.each([undefined, "ruby", "javascript"])( + "retains the first frame without a matching SDK frame on %s", + (platform) => { + const spans = getSpansFromProfile( + trace(), + profile({ platform }), + undefined, + 0, + 10, + new Set(), + ); + expect(spans).toMatchObject([ + { description: "app", op: "main", start_timestamp: 0, timestamp: 10 }, + ]); + }, + ); + + it("still removes SDK frames and all their ancestors", () => { + const spans = getSpansFromProfile( + trace(), + profile({ + platform: "javascript", + frames: [ + { module: "runtime", function: "run" }, + { module: "@sentry/core", function: "instrument" }, + { module: "app", function: "main" }, + ], + stacks: [[0, 1, 2]], + }), + undefined, + 0, + 10, + new Set(), + ); + expect(spans).toMatchObject([{ description: "app", op: "main" }]); + }); + + it.each([ + { + module: "@angular/core", + function: "render@component", + lineno: 12, + colno: 3, + }, + { + filename: "user@app.ts", + function: "render@component", + lineno: 12, + colno: 3, + }, + ])("preserves @ characters in module and function names", (frame) => { + const spans = getSpansFromProfile( + trace(), + profile({ platform: "javascript", frames: [frame] }), + undefined, + 0, + 10, + new Set(), + ); + expect(spans).toMatchObject([ + { + description: frame.module ?? frame.filename, + op: "render@component:12:3", + }, + ]); + }); +}); + +describe("profile graft thread selection", () => { + it("keeps distinct threads with identical names and frames separate", () => { + const spans = getSpansFromProfile( + trace(), + profile({ + thread_metadata: { + main: { name: "Thread" }, + worker: { name: "Thread" }, + }, + samples: [ + { thread_id: "main", stack_id: 0, start_timestamp: 0 }, + { thread_id: "worker", stack_id: 0, start_timestamp: 5 }, + { thread_id: "main", stack_id: 0, start_timestamp: 10 }, + { thread_id: "worker", stack_id: 0, start_timestamp: 15 }, + ], + }), + undefined, + 0, + 20, + new Set(["worker"]), + ); + expect(spans).toMatchObject([ + { + data: { thread_id: "main" }, + children: [{ op: "main", start_timestamp: 0, timestamp: 10 }], + }, + { + data: { thread_id: "worker" }, + children: [{ op: "main", start_timestamp: 5, timestamp: 15 }], + }, + { + data: { thread_id: "main" }, + children: [{ op: "main", start_timestamp: 10, timestamp: 20 }], + }, + { + data: { thread_id: "worker" }, + children: [{ op: "main", start_timestamp: 15, timestamp: 20 }], + }, + ]); + }); + + it("still consolidates consecutive samples and frames within one thread", () => { + const spans = getSpansFromProfile( + trace(), + profile({ + samples: [ + { thread_id: "main", stack_id: 0, start_timestamp: 0 }, + { thread_id: "main", stack_id: 0, start_timestamp: 5 }, + { thread_id: "main", stack_id: 0, start_timestamp: 10 }, + ], + }), + undefined, + 0, + 20, + new Set(), + ); + expect(spans).toMatchObject([ + { op: "main", start_timestamp: 0, timestamp: 20 }, + ]); + }); + + it("keeps simultaneous and interleaved thread intervals through their own next samples", () => { + const target = trace(); + const spans = getSpansFromProfile( + target, + profile({ + frames: [ + { module: "app", function: "main" }, + { module: "app", function: "worker" }, + ], + stacks: [[0], [1]], + samples: [ + { thread_id: "main", stack_id: 0, start_timestamp: 0 }, + { thread_id: "worker", stack_id: 1, start_timestamp: 0 }, + { thread_id: "main", stack_id: 0, start_timestamp: 10 }, + { thread_id: "worker", stack_id: 1, start_timestamp: 15 }, + ], + }), + undefined, + 0, + 20, + new Set(["worker"]), + ); + expect(spans).toMatchObject([ + { + description: "Thread main", + children: [{ op: "main", start_timestamp: 0, timestamp: 10 }], + }, + { + description: "Thread worker", + children: [{ op: "worker", start_timestamp: 0, timestamp: 15 }], + }, + { + description: "Thread main", + children: [{ op: "main", start_timestamp: 10, timestamp: 20 }], + }, + { + description: "Thread worker", + children: [{ op: "worker", start_timestamp: 15, timestamp: 20 }], + }, + ]); + // The trailing worker sample is flushed into both the tree and span lookup. + expect(target.spans.get(spans[3].span_id)).toBe(spans[3]); + }); + + it("keeps gap bounds when a same-thread next sample falls beyond the gap", () => { + const spans = getSpansFromProfile( + trace(), + profile({ + samples: [ + { thread_id: "main", stack_id: 0, start_timestamp: 0 }, + { thread_id: "main", stack_id: 0, start_timestamp: 4 }, + { thread_id: "main", stack_id: 0, start_timestamp: 12 }, + ], + }), + undefined, + 2, + 8, + new Set(), + ); + expect(spans).toEqual([]); + }); + + it.each(["previous", "next", "parent"])( + "uses the Sentry thread.id from the %s span", + (source) => { + const target = trace(); + const neighbor: Span = { + span_id: "neighbor", + start_timestamp: source === "next" ? 5 : 0, + timestamp: source === "next" ? 10 : 5, + data: { "thread.id": 7 }, + }; + if (source === "parent") target.data = { "thread.id": 7 }; + else target.spanTree = [neighbor]; + const start = source === "previous" ? 5 : 0; + graftProfileSpans( + target, + profile({ + platform: "javascript", + samples: [{ thread_id: "7", stack_id: 0, start_timestamp: start }], + }), + ); + expect( + target.spanTree.filter((span) => span.tags?.source === "profile"), + ).toMatchObject([{ description: "app", op: "main" }]); + expect(target.profileGrafted).toBe(true); + }, + ); + + it("retains active-thread and legacy threadId fallbacks", () => { + const target = trace(); + target.data = { threadId: "worker" }; + graftProfileSpans( + target, + profile({ + platform: "javascript", + frames: [ + { module: "app", function: "main" }, + { module: "app", function: "worker" }, + ], + stacks: [[0], [1]], + samples: [ + { thread_id: "main", stack_id: 0, start_timestamp: 0 }, + { thread_id: "worker", stack_id: 1, start_timestamp: 5 }, + ], + }), + ); + expect(target.spanTree.map((span) => span.children?.[0]?.op)).toEqual([ + "main", + "worker", + ]); + }); +}); diff --git a/packages/telemetry/src/telemetry/data/profiles.ts b/packages/spotlight/src/telemetry/data/profiles.ts similarity index 86% rename from packages/telemetry/src/telemetry/data/profiles.ts rename to packages/spotlight/src/telemetry/data/profiles.ts index 1104886d0..96f3f75e4 100644 --- a/packages/telemetry/src/telemetry/data/profiles.ts +++ b/packages/spotlight/src/telemetry/data/profiles.ts @@ -32,7 +32,9 @@ function consolidateSpans(trace: Trace, spans: Span[]): Span[] { if ( span && span.description === lastSpan.description && - span.op === lastSpan.op + span.op === lastSpan.op && + (span.op !== "Thread" || + span.data?.thread_id === lastSpan.data?.thread_id) ) { // Require the spans are sorted by start_timestamp lastSpan.timestamp = span.timestamp; @@ -105,6 +107,19 @@ export function getSpansFromProfile( const sentryFrameFilter = profile.platform && SENTRY_FRAME_FILTER_PER_PLATFORM[profile.platform]; + const nextSampleTimestamps: Array = []; + const nextTimestampByThread = new Map(); + for ( + let sampleIdx = profile.samples.length - 1; + sampleIdx >= 0; + sampleIdx-- + ) { + const sample = profile.samples[sampleIdx]; + nextSampleTimestamps[sampleIdx] = nextTimestampByThread.get( + sample.thread_id, + ); + nextTimestampByThread.set(sample.thread_id, sample.start_timestamp); + } // Try to fill in the gaps from profile data const fillerSpans: Span[] = []; for (let sampleIdx = 0; sampleIdx < profile.samples.length; sampleIdx++) { @@ -116,8 +131,7 @@ export function getSpansFromProfile( if (sampleTs < startTs || sampleTs > endTs) { continue; } - const nextSample = profile.samples[sampleIdx + 1]; - const timestamp = nextSample ? nextSample.start_timestamp : endTs; + const timestamp = nextSampleTimestamps[sampleIdx] ?? endTs; if (timestamp > endTs) { continue; @@ -162,7 +176,7 @@ export function getSpansFromProfile( const currentStack = profile.stacks[sample.stack_id]; const lastSentryFrameIdx = sentryFrameFilter ? currentStack.findLastIndex(sentryFrameFilter, profile.frames) - : 0; + : -1; for ( let frameIdxIdx = lastSentryFrameIdx + 1; frameIdxIdx < currentStack.length; @@ -171,7 +185,10 @@ export function getSpansFromProfile( const frame = profile.frames[currentStack[frameIdxIdx]]; // XXX: We may wanna skip frames that doesn't have `in_app` set to true // that said it's better to have this as a dynamic filter - const [description, op] = getFunctionNameFromFrame(frame).split("@"); + const description = + frame.module || frame.filename || frame.abs_path || ""; + // The separator follows the complete module; either component may contain @. + const op = getFunctionNameFromFrame(frame).slice(description.length + 1); const spanFromFrame: Span = { span_id: profileSpanId([...identity, "frame", frameIdxIdx]), parent_span_id: currentSpan.span_id, @@ -201,6 +218,13 @@ export function getSpansFromProfile( ); } +function getThreadId(span: Span | Trace | undefined): string | undefined { + const threadId = span?.data?.["thread.id"] ?? span?.data?.threadId; + return typeof threadId === "string" || typeof threadId === "number" + ? String(threadId) + : undefined; +} + /** * Modifies the spanTree in place recursively by adding spans from the * profile data where there are gaps in the trace data. @@ -234,9 +258,9 @@ export function graftProfileSpans( const startTs = span ? span.timestamp : parent.start_timestamp; const endTs = nextSpan ? nextSpan.start_timestamp : parent.timestamp; const threadIds = new Set([ - span?.data?.threadId, - nextSpan?.data?.threadId, - parent?.data?.threadId, + getThreadId(span), + getThreadId(nextSpan), + getThreadId(parent), ]); threadIds.delete(undefined); if (endTs - startTs > 0) { diff --git a/packages/telemetry/src/telemetry/envelope/codec.spec.ts b/packages/spotlight/src/telemetry/envelope/codec.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/envelope/codec.spec.ts rename to packages/spotlight/src/telemetry/envelope/codec.spec.ts diff --git a/packages/telemetry/src/telemetry/envelope/codec.ts b/packages/spotlight/src/telemetry/envelope/codec.ts similarity index 100% rename from packages/telemetry/src/telemetry/envelope/codec.ts rename to packages/spotlight/src/telemetry/envelope/codec.ts diff --git a/packages/telemetry/src/telemetry/envelope/payload.ts b/packages/spotlight/src/telemetry/envelope/payload.ts similarity index 100% rename from packages/telemetry/src/telemetry/envelope/payload.ts rename to packages/spotlight/src/telemetry/envelope/payload.ts diff --git a/packages/telemetry/src/telemetry/envelope/source.ts b/packages/spotlight/src/telemetry/envelope/source.ts similarity index 100% rename from packages/telemetry/src/telemetry/envelope/source.ts rename to packages/spotlight/src/telemetry/envelope/source.ts diff --git a/packages/telemetry/src/telemetry/envelope/userAgent.ts b/packages/spotlight/src/telemetry/envelope/userAgent.ts similarity index 100% rename from packages/telemetry/src/telemetry/envelope/userAgent.ts rename to packages/spotlight/src/telemetry/envelope/userAgent.ts diff --git a/packages/telemetry/src/telemetry/index.ts b/packages/spotlight/src/telemetry/index.ts similarity index 100% rename from packages/telemetry/src/telemetry/index.ts rename to packages/spotlight/src/telemetry/index.ts diff --git a/packages/telemetry/src/telemetry/inspection-models.spec.ts b/packages/spotlight/src/telemetry/inspection-models.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/inspection-models.spec.ts rename to packages/spotlight/src/telemetry/inspection-models.spec.ts diff --git a/packages/telemetry/src/telemetry/internal/engine.ts b/packages/spotlight/src/telemetry/internal/engine.ts similarity index 100% rename from packages/telemetry/src/telemetry/internal/engine.ts rename to packages/spotlight/src/telemetry/internal/engine.ts diff --git a/packages/telemetry/src/telemetry/internal/envelopes.ts b/packages/spotlight/src/telemetry/internal/envelopes.ts similarity index 100% rename from packages/telemetry/src/telemetry/internal/envelopes.ts rename to packages/spotlight/src/telemetry/internal/envelopes.ts diff --git a/packages/telemetry/src/telemetry/internal/readonly.spec.ts b/packages/spotlight/src/telemetry/internal/readonly.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/internal/readonly.spec.ts rename to packages/spotlight/src/telemetry/internal/readonly.spec.ts diff --git a/packages/telemetry/src/telemetry/internal/readonly.ts b/packages/spotlight/src/telemetry/internal/readonly.ts similarity index 100% rename from packages/telemetry/src/telemetry/internal/readonly.ts rename to packages/spotlight/src/telemetry/internal/readonly.ts diff --git a/packages/telemetry/src/telemetry/internal/results.ts b/packages/spotlight/src/telemetry/internal/results.ts similarity index 100% rename from packages/telemetry/src/telemetry/internal/results.ts rename to packages/spotlight/src/telemetry/internal/results.ts diff --git a/packages/telemetry/src/telemetry/internal/retention.ts b/packages/spotlight/src/telemetry/internal/retention.ts similarity index 100% rename from packages/telemetry/src/telemetry/internal/retention.ts rename to packages/spotlight/src/telemetry/internal/retention.ts diff --git a/packages/telemetry/src/telemetry/internal/types.ts b/packages/spotlight/src/telemetry/internal/types.ts similarity index 100% rename from packages/telemetry/src/telemetry/internal/types.ts rename to packages/spotlight/src/telemetry/internal/types.ts diff --git a/packages/telemetry/src/telemetry/internal/wireValues.ts b/packages/spotlight/src/telemetry/internal/wireValues.ts similarity index 100% rename from packages/telemetry/src/telemetry/internal/wireValues.ts rename to packages/spotlight/src/telemetry/internal/wireValues.ts diff --git a/packages/telemetry/src/telemetry/lifecycle.spec.ts b/packages/spotlight/src/telemetry/lifecycle.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/lifecycle.spec.ts rename to packages/spotlight/src/telemetry/lifecycle.spec.ts diff --git a/packages/telemetry/src/telemetry/log-traces.spec.ts b/packages/spotlight/src/telemetry/log-traces.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/log-traces.spec.ts rename to packages/spotlight/src/telemetry/log-traces.spec.ts diff --git a/packages/telemetry/src/telemetry/logger.ts b/packages/spotlight/src/telemetry/logger.ts similarity index 100% rename from packages/telemetry/src/telemetry/logger.ts rename to packages/spotlight/src/telemetry/logger.ts diff --git a/packages/telemetry/src/telemetry/profiles/model.ts b/packages/spotlight/src/telemetry/profiles/model.ts similarity index 100% rename from packages/telemetry/src/telemetry/profiles/model.ts rename to packages/spotlight/src/telemetry/profiles/model.ts diff --git a/packages/telemetry/src/telemetry/public.ts b/packages/spotlight/src/telemetry/public.ts similarity index 100% rename from packages/telemetry/src/telemetry/public.ts rename to packages/spotlight/src/telemetry/public.ts diff --git a/packages/telemetry/src/telemetry/queries.ts b/packages/spotlight/src/telemetry/queries.ts similarity index 100% rename from packages/telemetry/src/telemetry/queries.ts rename to packages/spotlight/src/telemetry/queries.ts diff --git a/packages/telemetry/src/telemetry/reader.spec.ts b/packages/spotlight/src/telemetry/reader.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/reader.spec.ts rename to packages/spotlight/src/telemetry/reader.spec.ts diff --git a/packages/telemetry/src/telemetry/reader.ts b/packages/spotlight/src/telemetry/reader.ts similarity index 100% rename from packages/telemetry/src/telemetry/reader.ts rename to packages/spotlight/src/telemetry/reader.ts diff --git a/packages/telemetry/src/telemetry/selectors/ai.ts b/packages/spotlight/src/telemetry/selectors/ai.ts similarity index 100% rename from packages/telemetry/src/telemetry/selectors/ai.ts rename to packages/spotlight/src/telemetry/selectors/ai.ts diff --git a/packages/telemetry/src/telemetry/selectors/database.ts b/packages/spotlight/src/telemetry/selectors/database.ts similarity index 100% rename from packages/telemetry/src/telemetry/selectors/database.ts rename to packages/spotlight/src/telemetry/selectors/database.ts diff --git a/packages/telemetry/src/telemetry/selectors/index.ts b/packages/spotlight/src/telemetry/selectors/index.ts similarity index 100% rename from packages/telemetry/src/telemetry/selectors/index.ts rename to packages/spotlight/src/telemetry/selectors/index.ts diff --git a/packages/spotlight/src/telemetry/selectors/profiles.test.ts b/packages/spotlight/src/telemetry/selectors/profiles.test.ts new file mode 100644 index 000000000..9999debb0 --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/profiles.test.ts @@ -0,0 +1,84 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "../../sdk/core.ts"; + +test("public profile summaries measure each thread until its own next sample", async () => { + const client = createTelemetryStore(); + try { + expect( + await client.ingestEnvelope([ + {}, + [ + [ + { type: "profile" }, + { + event_id: "profile", + version: "1", + timestamp: 1, + platform: "javascript", + transaction: { + id: "transaction", + trace_id: "trace-a", + name: "test", + active_thread_id: "main", + }, + profile: { + frames: [ + { module: "app", function: "recursive" }, + { module: "worker", function: "work" }, + { module: "app", function: "unbounded" }, + ], + stacks: [[0, 0], [1], [2]], + samples: [ + { + thread_id: "main", + stack_id: 0, + elapsed_since_start_ns: "0", + }, + { + thread_id: "worker", + stack_id: 1, + elapsed_since_start_ns: "5000000", + }, + { + thread_id: "main", + stack_id: 0, + elapsed_since_start_ns: "10000000", + }, + { + thread_id: "main", + stack_id: 2, + elapsed_since_start_ns: "20000000", + }, + { + thread_id: "worker", + stack_id: 2, + elapsed_since_start_ns: "35000000", + }, + ], + }, + }, + ], + ], + ]), + ).toMatchObject({ status: "processed" }); + expect(client.getProfileFunctionSummaries()).toEqual([ + { + name: "app@recursive", + totalDurationMs: 40, + sampleCount: 4, + traceIds: new Set(["trace-a"]), + }, + { + name: "worker@work", + totalDurationMs: 30, + sampleCount: 1, + traceIds: new Set(["trace-a"]), + }, + ]); + expect(client.getProfileFunctionSummaries({ traceId: "missing" })).toEqual( + [], + ); + } finally { + client.dispose(); + } +}); diff --git a/packages/telemetry/src/telemetry/selectors/profiles.ts b/packages/spotlight/src/telemetry/selectors/profiles.ts similarity index 61% rename from packages/telemetry/src/telemetry/selectors/profiles.ts rename to packages/spotlight/src/telemetry/selectors/profiles.ts index ef77a599a..41bd4297e 100644 --- a/packages/telemetry/src/telemetry/selectors/profiles.ts +++ b/packages/spotlight/src/telemetry/selectors/profiles.ts @@ -9,7 +9,7 @@ export type ProfileFunctionSummary = { traceIds: Set; }; -/** Inclusive frame durations in milliseconds, using each sample's interval to the next sample. */ +/** Inclusive frame durations in milliseconds, using intervals between samples of the same thread. */ export function getProfileFunctionSummaries( profilesByTraceId: ReadonlyMap, filter: SpanFilter = {}, @@ -17,15 +17,25 @@ export function getProfileFunctionSummaries( const aggregateCalls = new Map(); for (const [traceId, profile] of profilesByTraceId) { if (filter.traceId !== undefined && traceId !== filter.traceId) continue; - // The final sample has no known duration and is not included. + const durations: Array = []; + const nextTimestampByThread = new Map(); for ( - let sampleIdx = 0; - sampleIdx < profile.samples.length - 1; - sampleIdx++ + let sampleIdx = profile.samples.length - 1; + sampleIdx >= 0; + sampleIdx-- ) { const sample = profile.samples[sampleIdx]; - const duration = - profile.samples[sampleIdx + 1].start_timestamp - sample.start_timestamp; + const nextTimestamp = nextTimestampByThread.get(sample.thread_id); + if (nextTimestamp !== undefined) { + durations[sampleIdx] = nextTimestamp - sample.start_timestamp; + } + nextTimestampByThread.set(sample.thread_id, sample.start_timestamp); + } + // Each thread's final sample has no known duration and is not included. + for (let sampleIdx = 0; sampleIdx < profile.samples.length; sampleIdx++) { + const sample = profile.samples[sampleIdx]; + const duration = durations[sampleIdx]; + if (duration === undefined) continue; for (const frameIdx of profile.stacks[sample.stack_id]) { const name = getFunctionNameFromFrame(profile.frames[frameIdx]); const callData = aggregateCalls.get(name); diff --git a/packages/telemetry/src/telemetry/selectors/resources.ts b/packages/spotlight/src/telemetry/selectors/resources.ts similarity index 100% rename from packages/telemetry/src/telemetry/selectors/resources.ts rename to packages/spotlight/src/telemetry/selectors/resources.ts diff --git a/packages/telemetry/src/telemetry/selectors/selectors.test.ts b/packages/spotlight/src/telemetry/selectors/selectors.test.ts similarity index 86% rename from packages/telemetry/src/telemetry/selectors/selectors.test.ts rename to packages/spotlight/src/telemetry/selectors/selectors.test.ts index c99605b9a..81b29902a 100644 --- a/packages/telemetry/src/telemetry/selectors/selectors.test.ts +++ b/packages/spotlight/src/telemetry/selectors/selectors.test.ts @@ -3,6 +3,7 @@ import type { SentryProcessedProfile, Span } from "../types.ts"; import { getDatabaseQuerySummaries } from "./database.ts"; import { getProfileFunctionSummaries } from "./profiles.ts"; import { getResourceSummaries } from "./resources.ts"; +import { summarizeSpans } from "./spans.ts"; function span(overrides: Partial = {}): Span { return { @@ -44,7 +45,6 @@ describe("domain summaries", () => { first, span({ op: "http.client" }), second, - span({ description: " " }), span({ timestamp: Number.NaN }), span({ start_timestamp: Number.POSITIVE_INFINITY }), ]); @@ -168,3 +168,27 @@ it("averages known resource sizes across captured aliases without treating absen getResourceSummaries([span({ op: "resource.script" })])[0], ).not.toHaveProperty("averageSizeBytes"); }); + +it("groups unnamed spans by operation or ID without changing captured descriptions", () => { + const unnamed = span({ description: undefined }); + const blank = span({ span_id: "blank", description: " " }); + const noOperation = span({ span_id: "only-id", description: " ", op: " " }); + expect(summarizeSpans([unnamed, blank, noOperation])).toEqual([ + { + description: "db.sql.query", + count: 2, + totalDurationMs: 20, + averageDurationMs: 10, + spans: [unnamed, blank], + }, + { + description: "only-id", + count: 1, + totalDurationMs: 10, + averageDurationMs: 10, + spans: [noOperation], + }, + ]); + expect(unnamed.description).toBeUndefined(); + expect(blank.description).toBe(" "); +}); diff --git a/packages/telemetry/src/telemetry/selectors/spans.ts b/packages/spotlight/src/telemetry/selectors/spans.ts similarity index 86% rename from packages/telemetry/src/telemetry/selectors/spans.ts rename to packages/spotlight/src/telemetry/selectors/spans.ts index bb2b76496..e1e54a284 100644 --- a/packages/telemetry/src/telemetry/selectors/spans.ts +++ b/packages/spotlight/src/telemetry/selectors/spans.ts @@ -22,11 +22,12 @@ export function getSpans( .flatMap((trace) => Array.from(trace.spans.values())); } -/** Groups normalized spans by description, preserving their input order. Durations are milliseconds. */ +/** Groups spans by description, falling back to operation or ID. Durations are milliseconds. */ export function summarizeSpans(spans: Iterable): SpanSummary[] { const groups = new Map(); for (const span of spans) { - const description = span.description?.trim(); + const description = + span.description?.trim() || span.op?.trim() || span.span_id; if ( !description || !Number.isFinite(span.timestamp) || diff --git a/packages/telemetry/src/telemetry/selectors/vitalPages.ts b/packages/spotlight/src/telemetry/selectors/vitalPages.ts similarity index 100% rename from packages/telemetry/src/telemetry/selectors/vitalPages.ts rename to packages/spotlight/src/telemetry/selectors/vitalPages.ts diff --git a/packages/telemetry/src/telemetry/selectors/webVitals.ts b/packages/spotlight/src/telemetry/selectors/webVitals.ts similarity index 100% rename from packages/telemetry/src/telemetry/selectors/webVitals.ts rename to packages/spotlight/src/telemetry/selectors/webVitals.ts diff --git a/packages/telemetry/src/telemetry/serialization.spec.ts b/packages/spotlight/src/telemetry/serialization.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/serialization.spec.ts rename to packages/spotlight/src/telemetry/serialization.spec.ts diff --git a/packages/telemetry/src/telemetry/serialization.ts b/packages/spotlight/src/telemetry/serialization.ts similarity index 100% rename from packages/telemetry/src/telemetry/serialization.ts rename to packages/spotlight/src/telemetry/serialization.ts diff --git a/packages/spotlight/src/telemetry/store/helpers.ts b/packages/spotlight/src/telemetry/store/helpers.ts new file mode 100644 index 000000000..ac71d61bb --- /dev/null +++ b/packages/spotlight/src/telemetry/store/helpers.ts @@ -0,0 +1,16 @@ +import type { Span } from "../types.ts"; + +export function getAllSpansInTree(root: Span): Span[] { + const spans: Span[] = []; + const pending = [root]; + const visited = new Set(); + for (let span = pending.pop(); span; span = pending.pop()) { + if (visited.has(span)) continue; + visited.add(span); + spans.push(span); + for (let index = (span.children?.length ?? 0) - 1; index >= 0; index--) { + pending.push(span.children![index]); + } + } + return spans; +} diff --git a/packages/telemetry/src/telemetry/store/slices/envelopesSlice.ts b/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/slices/envelopesSlice.ts rename to packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts diff --git a/packages/telemetry/src/telemetry/store/slices/eventsSlice.ts b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/slices/eventsSlice.ts rename to packages/spotlight/src/telemetry/store/slices/eventsSlice.ts diff --git a/packages/telemetry/src/telemetry/store/slices/logsSlice.ts b/packages/spotlight/src/telemetry/store/slices/logsSlice.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/slices/logsSlice.ts rename to packages/spotlight/src/telemetry/store/slices/logsSlice.ts diff --git a/packages/telemetry/src/telemetry/store/slices/profilesSlice.ts b/packages/spotlight/src/telemetry/store/slices/profilesSlice.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/slices/profilesSlice.ts rename to packages/spotlight/src/telemetry/store/slices/profilesSlice.ts diff --git a/packages/telemetry/src/telemetry/store/slices/sdksSlice.ts b/packages/spotlight/src/telemetry/store/slices/sdksSlice.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/slices/sdksSlice.ts rename to packages/spotlight/src/telemetry/store/slices/sdksSlice.ts diff --git a/packages/telemetry/src/telemetry/store/slices/sharedSlice.ts b/packages/spotlight/src/telemetry/store/slices/sharedSlice.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/slices/sharedSlice.ts rename to packages/spotlight/src/telemetry/store/slices/sharedSlice.ts diff --git a/packages/telemetry/src/telemetry/store/slices/subscriptionsSlice.ts b/packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/slices/subscriptionsSlice.ts rename to packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts diff --git a/packages/telemetry/src/telemetry/store/slices/tracesSlice.ts b/packages/spotlight/src/telemetry/store/slices/tracesSlice.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/slices/tracesSlice.ts rename to packages/spotlight/src/telemetry/store/slices/tracesSlice.ts diff --git a/packages/telemetry/src/telemetry/store/types.ts b/packages/spotlight/src/telemetry/store/types.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/types.ts rename to packages/spotlight/src/telemetry/store/types.ts diff --git a/packages/telemetry/src/telemetry/store/utils.ts b/packages/spotlight/src/telemetry/store/utils.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/utils.ts rename to packages/spotlight/src/telemetry/store/utils.ts diff --git a/packages/telemetry/src/telemetry/store/utils/logProcessor.ts b/packages/spotlight/src/telemetry/store/utils/logProcessor.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/utils/logProcessor.ts rename to packages/spotlight/src/telemetry/store/utils/logProcessor.ts diff --git a/packages/telemetry/src/telemetry/store/utils/profileChunkProcessor.ts b/packages/spotlight/src/telemetry/store/utils/profileChunkProcessor.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/utils/profileChunkProcessor.ts rename to packages/spotlight/src/telemetry/store/utils/profileChunkProcessor.ts diff --git a/packages/telemetry/src/telemetry/store/utils/profileProcessor.ts b/packages/spotlight/src/telemetry/store/utils/profileProcessor.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/utils/profileProcessor.ts rename to packages/spotlight/src/telemetry/store/utils/profileProcessor.ts diff --git a/packages/telemetry/src/telemetry/store/utils/spanProcessor.ts b/packages/spotlight/src/telemetry/store/utils/spanProcessor.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/utils/spanProcessor.ts rename to packages/spotlight/src/telemetry/store/utils/spanProcessor.ts diff --git a/packages/telemetry/src/telemetry/store/utils/traceInitializer.ts b/packages/spotlight/src/telemetry/store/utils/traceInitializer.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/utils/traceInitializer.ts rename to packages/spotlight/src/telemetry/store/utils/traceInitializer.ts diff --git a/packages/telemetry/src/telemetry/store/utils/traceProcessor.ts b/packages/spotlight/src/telemetry/store/utils/traceProcessor.ts similarity index 100% rename from packages/telemetry/src/telemetry/store/utils/traceProcessor.ts rename to packages/spotlight/src/telemetry/store/utils/traceProcessor.ts diff --git a/packages/telemetry/src/telemetry/topology.spec.ts b/packages/spotlight/src/telemetry/topology.spec.ts similarity index 100% rename from packages/telemetry/src/telemetry/topology.spec.ts rename to packages/spotlight/src/telemetry/topology.spec.ts diff --git a/packages/telemetry/src/telemetry/types.ts b/packages/spotlight/src/telemetry/types.ts similarity index 100% rename from packages/telemetry/src/telemetry/types.ts rename to packages/spotlight/src/telemetry/types.ts diff --git a/packages/telemetry/src/telemetry/utils/profileUtils.ts b/packages/spotlight/src/telemetry/utils/profileUtils.ts similarity index 100% rename from packages/telemetry/src/telemetry/utils/profileUtils.ts rename to packages/spotlight/src/telemetry/utils/profileUtils.ts diff --git a/packages/telemetry/src/telemetry/utils/sdkToPlatform.ts b/packages/spotlight/src/telemetry/utils/sdkToPlatform.ts similarity index 100% rename from packages/telemetry/src/telemetry/utils/sdkToPlatform.ts rename to packages/spotlight/src/telemetry/utils/sdkToPlatform.ts diff --git a/packages/telemetry/src/telemetry/utils/sentry.ts b/packages/spotlight/src/telemetry/utils/sentry.ts similarity index 100% rename from packages/telemetry/src/telemetry/utils/sentry.ts rename to packages/spotlight/src/telemetry/utils/sentry.ts diff --git a/packages/telemetry/src/telemetry/utils/stringifyValue.test.ts b/packages/spotlight/src/telemetry/utils/stringifyValue.test.ts similarity index 100% rename from packages/telemetry/src/telemetry/utils/stringifyValue.test.ts rename to packages/spotlight/src/telemetry/utils/stringifyValue.test.ts diff --git a/packages/telemetry/src/telemetry/utils/stringifyValue.ts b/packages/spotlight/src/telemetry/utils/stringifyValue.ts similarity index 100% rename from packages/telemetry/src/telemetry/utils/stringifyValue.ts rename to packages/spotlight/src/telemetry/utils/stringifyValue.ts diff --git a/packages/telemetry/src/telemetry/utils/traces.ts b/packages/spotlight/src/telemetry/utils/traces.ts similarity index 100% rename from packages/telemetry/src/telemetry/utils/traces.ts rename to packages/spotlight/src/telemetry/utils/traces.ts diff --git a/packages/telemetry/src/telemetry/utils/webVitals.ts b/packages/spotlight/src/telemetry/utils/webVitals.ts similarity index 100% rename from packages/telemetry/src/telemetry/utils/webVitals.ts rename to packages/spotlight/src/telemetry/utils/webVitals.ts diff --git a/packages/telemetry/src/telemetry/uuid.ts b/packages/spotlight/src/telemetry/uuid.ts similarity index 100% rename from packages/telemetry/src/telemetry/uuid.ts rename to packages/spotlight/src/telemetry/uuid.ts diff --git a/packages/telemetry/tests/types/sdk-core.ts b/packages/spotlight/tests/types/sdk-core.ts similarity index 99% rename from packages/telemetry/tests/types/sdk-core.ts rename to packages/spotlight/tests/types/sdk-core.ts index 36b3db696..d82606d90 100644 --- a/packages/telemetry/tests/types/sdk-core.ts +++ b/packages/spotlight/tests/types/sdk-core.ts @@ -9,7 +9,7 @@ import { serializeSnapshot, serializeSummaries, serializeTrace, -} from "@sentry/telemetry"; +} from "@sentry/spotlight"; const telemetry = createTelemetryStore(); const queries: readonly QuerySummary[] = telemetry.getDatabaseQuerySummaries({ diff --git a/packages/telemetry/tests/types/sdk-node.ts b/packages/spotlight/tests/types/sdk-node.ts similarity index 91% rename from packages/telemetry/tests/types/sdk-node.ts rename to packages/spotlight/tests/types/sdk-node.ts index 296fac730..318ba72d1 100644 --- a/packages/telemetry/tests/types/sdk-node.ts +++ b/packages/spotlight/tests/types/sdk-node.ts @@ -1,8 +1,8 @@ -import type { IngestResult, TelemetryReader } from "@sentry/telemetry"; +import type { IngestResult, TelemetryReader } from "@sentry/spotlight"; import { type TelemetryPayload, createTelemetryPipeline, -} from "@sentry/telemetry/node"; +} from "@sentry/spotlight/node"; const pipeline = createTelemetryPipeline({ maxPending: 4, diff --git a/packages/telemetry/tests/types/tsconfig.sdk-core.json b/packages/spotlight/tests/types/tsconfig.sdk-core.json similarity index 100% rename from packages/telemetry/tests/types/tsconfig.sdk-core.json rename to packages/spotlight/tests/types/tsconfig.sdk-core.json diff --git a/packages/telemetry/tests/types/tsconfig.sdk-node.json b/packages/spotlight/tests/types/tsconfig.sdk-node.json similarity index 100% rename from packages/telemetry/tests/types/tsconfig.sdk-node.json rename to packages/spotlight/tests/types/tsconfig.sdk-node.json diff --git a/packages/telemetry/tsconfig.build.json b/packages/spotlight/tsconfig.build.json similarity index 100% rename from packages/telemetry/tsconfig.build.json rename to packages/spotlight/tsconfig.build.json diff --git a/packages/telemetry/tsconfig.json b/packages/spotlight/tsconfig.json similarity index 100% rename from packages/telemetry/tsconfig.json rename to packages/spotlight/tsconfig.json diff --git a/packages/telemetry/vitest.config.ts b/packages/spotlight/vitest.config.ts similarity index 100% rename from packages/telemetry/vitest.config.ts rename to packages/spotlight/vitest.config.ts diff --git a/packages/telemetry/src/telemetry/store/helpers.ts b/packages/telemetry/src/telemetry/store/helpers.ts deleted file mode 100644 index d79423301..000000000 --- a/packages/telemetry/src/telemetry/store/helpers.ts +++ /dev/null @@ -1,8 +0,0 @@ -import type { Span } from "../types.ts"; - -export function getAllSpansInTree(root: Span): Span[] { - const spans = [root]; - return root.children - ? spans.concat(root.children.flatMap(getAllSpansInTree)) - : spans; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0ffc46e7..06f0c07a9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -352,9 +352,9 @@ importers: '@radix-ui/react-tooltip': specifier: ^1.4.1 version: 1.4.1(@types/react-dom@19.3.0(@types/react@19.3.0))(@types/react@19.3.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) - '@sentry/telemetry': + '@sentry/spotlight': specifier: workspace:* - version: link:../../packages/telemetry + version: link:../../packages/spotlight '@shikijs/core': specifier: ^3.23.0 version: 3.23.0 @@ -488,9 +488,9 @@ importers: '@sentry/symbolic': specifier: 13.7.0 version: 13.7.0 - '@sentry/telemetry': + '@sentry/spotlight': specifier: workspace:* - version: link:../telemetry + version: link:../spotlight '@sentry/toolkit-core': specifier: workspace:* version: link:../toolkit-core @@ -1021,7 +1021,7 @@ importers: specifier: 'catalog:' version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@22.16.0)(@vitest/coverage-v8@4.1.11)(jsdom@28.1.0)(msw@2.10.2(@types/node@22.16.0)(typescript@5.9.3))(vite@8.3.3(@types/node@22.16.0)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.13)(yaml@2.9.0)) - packages/telemetry: + packages/spotlight: dependencies: '@sentry/core': specifier: ^10.63.0 From ccc1eb829f7e9d2659dc702e1da63cc9c600bb87 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Wed, 28 Oct 2026 18:27:25 +0530 Subject: [PATCH 03/15] fix(spotlight): Keep streamed profiles and AI reads coherent Co-Authored-By: Codex --- .../cli/src/commands/local/session-store.ts | 2 +- .../test/commands/local/session-store.test.ts | 29 ++ packages/spotlight/README.md | 6 + .../src/sdk/node-history-retention.spec.ts | 143 ++++++ packages/spotlight/src/sdk/node.ts | 8 +- .../src/telemetry/ai-regressions.spec.ts | 323 +++++++++++++ .../src/telemetry/ai/finishReason.ts | 20 + packages/spotlight/src/telemetry/ai/genAI.ts | 11 +- .../spotlight/src/telemetry/ai/vercelAISDK.ts | 186 +++----- .../src/telemetry/data/profiles.test.ts | 81 +++- .../spotlight/src/telemetry/data/profiles.ts | 149 ++++-- .../src/telemetry/profile-depth.spec.ts | 115 +++++ .../src/telemetry/profiles-streaming.spec.ts | 429 ++++++++++++++++++ .../telemetry/store/slices/envelopesSlice.ts | 5 +- .../src/telemetry/store/slices/eventsSlice.ts | 120 ++--- 15 files changed, 1385 insertions(+), 242 deletions(-) create mode 100644 packages/spotlight/src/sdk/node-history-retention.spec.ts create mode 100644 packages/spotlight/src/telemetry/ai/finishReason.ts create mode 100644 packages/spotlight/src/telemetry/profile-depth.spec.ts create mode 100644 packages/spotlight/src/telemetry/profiles-streaming.spec.ts diff --git a/packages/cli/src/commands/local/session-store.ts b/packages/cli/src/commands/local/session-store.ts index 3f0c62e16..baf1ccfc5 100644 --- a/packages/cli/src/commands/local/session-store.ts +++ b/packages/cli/src/commands/local/session-store.ts @@ -173,7 +173,7 @@ export function createLocalSessionStore({ const session = resolveStored(reference); await session.retention.resize(value); return toPublicSession( - update(resolveStored(reference), { maxEnvelopes: value }), + update(resolveStored(session.id), { maxEnvelopes: value }), ); }, close(reference) { diff --git a/packages/cli/test/commands/local/session-store.test.ts b/packages/cli/test/commands/local/session-store.test.ts index 2f68715be..fd9b02a2d 100644 --- a/packages/cli/test/commands/local/session-store.test.ts +++ b/packages/cli/test/commands/local/session-store.test.ts @@ -167,3 +167,32 @@ describe("LocalSessionStore", () => { unsubscribe(); }); }); + +test("retention by label remains bound to its session when the label becomes ambiguous", async () => { + const store = createLocalSessionStore(); + const original = store.create({ + label: "web", + cwd: "/first", + maxEnvelopes: 2, + }); + const resize = store.setMaxEnvelopes("web", 4); + const other = store.create({ label: "web", cwd: "/second", maxEnvelopes: 3 }); + store.close(original.id); + try { + await expect(resize).resolves.toMatchObject({ + id: original.id, + maxEnvelopes: 4, + state: "retained", + }); + expect(store.resolve(original.id)).toMatchObject({ + maxEnvelopes: 4, + state: "retained", + }); + expect(store.getRetention(original.id).getSettings().maxEnvelopes).toBe(4); + expect(store.resolve(other.id).maxEnvelopes).toBe(3); + expect(store.getRetention(other.id).getSettings().maxEnvelopes).toBe(3); + } finally { + store.getBuffer(original.id).dispose(); + store.getBuffer(other.id).dispose(); + } +}); diff --git a/packages/spotlight/README.md b/packages/spotlight/README.md index 81d9d5606..f9eab2cc2 100644 --- a/packages/spotlight/README.md +++ b/packages/spotlight/README.md @@ -168,6 +168,12 @@ invalidates in-flight/queued work. `prune()` explicitly enforces retention. Eviction rebuilds derived state from retained envelopes, with cost proportional to retained telemetry. Choose smaller limits for large profiles or heavy streams. +### AI usage in incomplete traces + +Vercel AI token summaries reconcile each span's reported usage with its observed child subtrees, taking the larger subtotal rather than adding an aggregate parent twice. This supports both legacy last-step values and newer aggregate spans. If spans are missing, the result is the strongest observed subtotal; it cannot reconstruct unseen model calls. Invalid/non-finite values do not contribute to these subtotals. + +Continuous profile chunks refresh the merged profile and its derived spans. Profile grafting uses the bounds of observed real spans; later context-only errors or logs can extend trace timestamps without inventing extra profiled work. + ## Pure processing helpers The same portable entry also exports models that can be computed without retaining a telemetry session: diff --git a/packages/spotlight/src/sdk/node-history-retention.spec.ts b/packages/spotlight/src/sdk/node-history-retention.spec.ts new file mode 100644 index 000000000..3638ac085 --- /dev/null +++ b/packages/spotlight/src/sdk/node-history-retention.spec.ts @@ -0,0 +1,143 @@ +import { expect, test, vi } from "vitest"; +import { createTelemetryPipeline } from "./node.ts"; + +function body(eventId: string): string { + return `{}\n{"type":"event"}\n${JSON.stringify({ event_id: eventId, timestamp: 1 })}`; +} + +async function retainedDelivery() { + vi.setSystemTime(1000); + const pipeline = createTelemetryPipeline({ retention: { maxAgeMs: 15 } }); + await pipeline.ingestPayload({ body: body("expired"), deliveryId: "old" }); + vi.setSystemTime(1010); + await pipeline.ingestPayload({ body: body("original"), deliveryId: "same" }); + // Move the clock without running the expiry timer. Admission must prune the + // older envelope while retaining the original delivery that is redispatched. + vi.setSystemTime(1021); + return pipeline; +} + +test("an eviction during duplicate admission cannot replace the original decoded delivery", async () => { + vi.useFakeTimers(); + const pipeline = await retainedDelivery(); + try { + const observations: { type: string; raw?: string }[] = []; + pipeline.subscribe((_snapshot, change) => { + observations.push({ + type: change.type, + raw: pipeline.getDecodedPayload("same")?.body.toString(), + }); + }); + expect( + await pipeline.ingestPayload({ + body: body("replacement"), + deliveryId: "same", + }), + ).toMatchObject({ status: "ignored", reason: "duplicate" }); + expect(observations).toEqual([{ type: "evict", raw: body("original") }]); + expect(pipeline.getDecodedPayload("same")?.body.toString()).toBe( + body("original"), + ); + expect(pipeline.getEvent("original")).toBeDefined(); + expect(pipeline.getEvent("replacement")).toBeUndefined(); + } finally { + pipeline.dispose(); + vi.useRealTimers(); + } +}); + +test("aborting a redispatch across retention rebuild awaits never commits its staged bytes", async () => { + vi.useFakeTimers(); + let observedAbortedEviction = false; + try { + // Exercise cancellation on both sides of the rebuild's promise boundaries, + // including the interval after pruning succeeds and before admission resumes. + for (let abortAfter = 0; abortAfter <= 12; abortAfter++) { + const pipeline = await retainedDelivery(); + try { + const observations: { type: string; raw?: string }[] = []; + pipeline.subscribe((_snapshot, change) => { + observations.push({ + type: change.type, + raw: pipeline.getDecodedPayload("same")?.body.toString(), + }); + }); + const controller = new AbortController(); + const pending = pipeline.ingestPayload({ + body: body("replacement"), + deliveryId: "same", + signal: controller.signal, + }); + for (let turn = 0; turn < abortAfter; turn++) await Promise.resolve(); + controller.abort(); + const result = await pending; + expect(result.status).toBe("ignored"); + if ( + result.status === "ignored" && + result.reason === "aborted" && + observations.some((change) => change.type === "evict") + ) + observedAbortedEviction = true; + for (const observation of observations) + expect(observation.raw).toBe(body("original")); + expect(pipeline.getDecodedPayload("same")?.body.toString()).toBe( + body("original"), + ); + expect(pipeline.getEvent("replacement")).toBeUndefined(); + } finally { + pipeline.dispose(); + } + } + expect(observedAbortedEviction).toBe(true); + } finally { + vi.useRealTimers(); + } +}); + +test.each([ + { status: "processed", payload: body("newest") }, + { status: "ignored", payload: '{}\n{"type":"attachment","length":5}\nhello' }, +] as const)( + "$status payload bytes commit before observers even when byte retention immediately evicts the envelope", + async ({ status, payload }) => { + const pipeline = createTelemetryPipeline({ + retention: { maxBytes: 1 }, + decodedHistory: { maxEntries: 1 }, + }); + try { + await pipeline.ingestPayload({ body: body("old"), deliveryId: "old" }); + const observations: { + envelopeCount: number; + old?: string; + newest?: string; + }[] = []; + pipeline.subscribe((snapshot, change) => { + if (change.type !== "ingest") return; + observations.push({ + envelopeCount: snapshot.envelopes.length, + old: pipeline.getDecodedPayload("old")?.body.toString(), + newest: pipeline.getDecodedPayload("newest")?.body.toString(), + }); + }); + const result = await pipeline.ingestPayload({ + body: payload, + deliveryId: "newest", + }); + expect(result).toMatchObject({ + status, + retained: false, + envelopeId: "newest", + }); + if (status === "ignored") + expect(result).toMatchObject({ reason: "unsupported_items" }); + expect(observations).toEqual([ + { envelopeCount: 0, old: undefined, newest: payload }, + ]); + expect(pipeline.getDecodedPayload("newest")?.body.toString()).toBe( + payload, + ); + } finally { + pipeline.dispose(); + } + }, +); diff --git a/packages/spotlight/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts index cd04bdbbf..e68ac84c4 100644 --- a/packages/spotlight/src/sdk/node.ts +++ b/packages/spotlight/src/sdk/node.ts @@ -133,9 +133,11 @@ export function createTelemetryPipeline( } // Register before public listeners so decoded history commits with the core snapshot. - telemetry.subscribe((snapshot) => { - for (const envelope of snapshot.envelopes) { - const envelopeId = stringifyValue(envelope[0].__spotlight_envelope_id); + telemetry.subscribe((_snapshot, change) => { + // An eviction can publish an existing ID while its redispatch is staged. + // Only successful ingestion authorizes replacing decoded delivery bytes. + if (change.type !== "ingest") return; + for (const envelopeId of change.envelopeIds) { const payload = stagedPayloads.get(envelopeId); if (!payload) continue; stagedPayloads.delete(envelopeId); diff --git a/packages/spotlight/src/telemetry/ai-regressions.spec.ts b/packages/spotlight/src/telemetry/ai-regressions.spec.ts index 5fc0b6f31..156bdcf4d 100644 --- a/packages/spotlight/src/telemetry/ai-regressions.spec.ts +++ b/packages/spotlight/src/telemetry/ai-regressions.spec.ts @@ -52,6 +52,329 @@ const calls = [ { toolCallId: "weather-call", toolName: "weather", args: { city: "Pune" } }, ]; +test.each([ + { value: '["stop"]', expected: "stop" }, + { value: '["stop","length"]', expected: "stop" }, + { value: ["length"], expected: "length" }, + { value: "stop", expected: "stop" }, + { value: "[]", expected: undefined }, + { value: [], expected: undefined }, + { value: '["stop"', expected: undefined }, + { value: "null", expected: undefined }, + { value: [null], expected: undefined }, + { value: " ", expected: undefined }, +])("AI queries normalize finish reason $value", async ({ value, expected }) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "gen_ai.response.finish_reasons": value, + "gen_ai.response.text": "Available response", + }), + ]); + expect(store.getAITraces()[0]?.response).toMatchObject({ + text: "Available response", + }); + expect(store.getAITraces()[0]?.response?.finishReason).toBe(expected); + } finally { + store.dispose(); + } +}); + +test.each([ + { value: '["stop"]', expected: "stop" }, + { value: ["length"], expected: "length" }, + { value: "stop", expected: "stop" }, + { value: "[]", expected: undefined }, + { value: [], expected: undefined }, + { value: '["stop"', expected: undefined }, + { value: "null", expected: undefined }, + { value: [null], expected: undefined }, + { value: " ", expected: undefined }, +])( + "Vercel AI queries normalize finish reason $value", + async ({ value, expected }) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "gen_ai.response.finish_reasons": value, + "gen_ai.response.text": "Available response", + }), + ]); + expect(store.getAITraces()[0]?.response?.text).toBe("Available response"); + expect(store.getAITraces()[0]?.response?.finishReason).toBe(expected); + } finally { + store.dispose(); + } + }, +); + +test("Vercel AI queries prioritize the legacy singular finish reason", async () => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "vercel.ai.response.finishReason": "tool-calls", + "gen_ai.response.finish_reasons": '["stop"]', + }), + ]); + expect(store.getAITraces()[0]?.response?.finishReason).toBe("tool-calls"); + } finally { + store.dispose(); + } +}); + +test.each([undefined, 30])( + "AI queries keep token subtotals finite when child sums overflow, root=%j", + async (rootTokens) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "gen_ai.usage.input_tokens": rootTokens, + }), + span( + "first", + { + "gen_ai.usage.input_tokens": 1e308, + "gen_ai.usage.output_tokens": 2, + }, + "root", + ), + span( + "second", + { + "gen_ai.usage.input_tokens": 1e308, + "gen_ai.usage.output_tokens": 3, + }, + "root", + ), + ]); + expect(store.getAITraces()[0]?.promptTokens).toBe(rootTokens); + expect(store.getAITraces()[0]?.metadata.promptTokens).toBe(rootTokens); + expect(store.getAITraces()[0]?.completionTokens).toBe(5); + } finally { + store.dispose(); + } + }, +); + +test.each([ + { + field: "vercel.ai.operationId", + operation: "ai.generateText", + tool: "ai.toolCall", + }, + { field: "operation.name", operation: "ai.streamText", tool: "ai.toolCall" }, + { + field: "gen_ai.operation.name", + operation: "invoke_agent", + tool: "execute_tool", + }, +])( + "AI queries retain root identity with $field tool descendants", + async ({ field, operation, tool }) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + { + ...span( + "root", + { [field]: operation, "vercel.ai.model.id": "model" }, + "transaction", + "gen_ai.invoke_agent", + ), + description: "Plan itinerary", + }, + span("model", { "gen_ai.request.model": "model" }, "root"), + span( + "tool", + { [field]: tool, "gen_ai.tool.name": "weather" }, + "model", + "gen_ai.execute_tool", + ), + ]); + expect(store.getAITraces()).toHaveLength(1); + expect(store.getAITraces()[0]).toMatchObject({ + operation, + name: "Plan itinerary", + hasToolCall: true, + }); + } finally { + store.dispose(); + } + }, +); + +test("AI queries name a Vercel tool root from its own tool name", async () => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span( + "root", + { + "vercel.ai.operationId": "ai.toolCall", + "gen_ai.tool.name": "weather", + }, + "transaction", + "gen_ai.execute_tool", + ), + ]); + expect(store.getAITraces()[0]).toMatchObject({ + operation: "ai.toolCall", + name: "weather", + hasToolCall: true, + }); + } finally { + store.dispose(); + } +}); + +test.each([ + { + label: "modern aggregate", + root: { "gen_ai.usage.input_tokens": 30, "gen_ai.usage.output_tokens": 7 }, + expectedInput: 30, + expectedOutput: 7, + }, + { + label: "zero root with observed model usage", + root: { "gen_ai.usage.input_tokens": 0, "gen_ai.usage.output_tokens": 0 }, + expectedInput: 30, + expectedOutput: 7, + }, + { + label: "normalized legacy last-step root", + root: { "gen_ai.usage.input_tokens": 20, "gen_ai.usage.output_tokens": 5 }, + expectedInput: 30, + expectedOutput: 7, + }, + { + label: "aggregate with partial child coverage", + root: { + "gen_ai.usage.input_tokens": 100, + "gen_ai.usage.output_tokens": 80, + }, + expectedInput: 100, + expectedOutput: 80, + }, + { + label: "legacy last-step root", + root: { + "vercel.ai.usage.promptTokens": 20, + "vercel.ai.usage.completionTokens": 5, + }, + expectedInput: 30, + expectedOutput: 7, + }, + { + label: "missing root usage", + root: {}, + expectedInput: 30, + expectedOutput: 7, + }, + { + label: "malformed root usage", + root: { + "gen_ai.usage.input_tokens": "bad", + "gen_ai.usage.output_tokens": null, + }, + expectedInput: 30, + expectedOutput: 7, + }, +])( + "AI queries count Vercel multi-step usage with $label", + async ({ root, expectedInput, expectedOutput }) => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span( + "root", + { "vercel.ai.operationId": "ai.generateText", ...root }, + "transaction", + "gen_ai.invoke_agent", + ), + span( + "wrapper", + { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.usage.promptTokens": 20, + "vercel.ai.usage.completionTokens": 5, + }, + "root", + "gen_ai.invoke_agent", + ), + span( + "first-model", + { + "vercel.ai.operationId": "ai.generateText.doGenerate", + "gen_ai.usage.input_tokens": 10, + "gen_ai.usage.output_tokens": 2, + }, + "wrapper", + "gen_ai.generate_text", + ), + span( + "second-model", + { + "operation.name": "ai.generateText.doGenerate", + "vercel.ai.usage.promptTokens": "20", + "vercel.ai.usage.completionTokens": "5", + }, + "wrapper", + "gen_ai.generate_text", + ), + span( + "invalid-model", + { + "vercel.ai.operationId": "ai.generateText.doGenerate", + "gen_ai.usage.input_tokens": "NaN", + "gen_ai.usage.output_tokens": false, + }, + "wrapper", + "gen_ai.generate_text", + ), + ]); + expect(store.getAITraces()[0]).toMatchObject({ + promptTokens: expectedInput, + completionTokens: expectedOutput, + tokensDisplay: `${expectedInput} / ${expectedOutput}`, + metadata: { + promptTokens: expectedInput, + completionTokens: expectedOutput, + }, + }); + } finally { + store.dispose(); + } + }, +); + +test("AI queries retain zero model usage and omit malformed Vercel token counts", async () => { + const store = createTelemetryStore(); + try { + await ingestSpans(store, [ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "gen_ai.usage.input_tokens": 0, + "gen_ai.usage.output_tokens": "Infinity", + }), + ]); + expect(store.getAITraces()[0]).toMatchObject({ + promptTokens: 0, + tokensDisplay: "0 / ?", + metadata: { promptTokens: 0 }, + }); + expect(store.getAITraces()[0]?.completionTokens).toBeUndefined(); + expect(store.getAITraces()[0]?.metadata.completionTokens).toBeUndefined(); + } finally { + store.dispose(); + } +}); + test.each(["generic", "vercel"])( "AI queries preserve descendant response order and shallowest %s roots", async (provider) => { diff --git a/packages/spotlight/src/telemetry/ai/finishReason.ts b/packages/spotlight/src/telemetry/ai/finishReason.ts new file mode 100644 index 000000000..a7dc0c4d7 --- /dev/null +++ b/packages/spotlight/src/telemetry/ai/finishReason.ts @@ -0,0 +1,20 @@ +/** Decode the convention's JSON array while retaining older plain reasons. */ +export function extractFinishReason( + finishReasons: unknown, +): string | undefined { + if (Array.isArray(finishReasons)) { + const first = finishReasons[0]; + return typeof first === "string" && first.trim() ? first.trim() : undefined; + } + if (typeof finishReasons === "string" && finishReasons.trim()) { + const text = finishReasons.trim(); + try { + const parsed: unknown = JSON.parse(text); + return Array.isArray(parsed) ? extractFinishReason(parsed) : undefined; + } catch { + // Older SDKs also send a plain reason; truncated JSON is not one. + return ["[", "{", '"'].includes(text[0]) ? undefined : text; + } + } + return undefined; +} diff --git a/packages/spotlight/src/telemetry/ai/genAI.ts b/packages/spotlight/src/telemetry/ai/genAI.ts index d8bc8a63b..af2be8a51 100644 --- a/packages/spotlight/src/telemetry/ai/genAI.ts +++ b/packages/spotlight/src/telemetry/ai/genAI.ts @@ -1,6 +1,7 @@ import { stringifyValue } from "../utils/stringifyValue.ts"; import { getAllSpansInTree } from "../store/helpers.ts"; import type { AILibraryHandler, AIToolCall, AITrace, Span } from "../types.ts"; +import { extractFinishReason } from "./finishReason.ts"; // Generic handler for the standardized `gen_ai.*` span conventions. Any SDK // that follows sentry-conventions (sentry-python's PydanticAI/OpenAI/Anthropic @@ -317,16 +318,6 @@ function extractPromptData(span: Span, trace: AITrace) { }; } -function extractFinishReason(finishReasons: unknown): string | undefined { - if (Array.isArray(finishReasons) && finishReasons.length > 0) { - return stringifyValue(finishReasons[0]); - } - if (typeof finishReasons === "string" && finishReasons) { - return finishReasons; - } - return undefined; -} - function extractResponseData( span: Span, trace: AITrace, diff --git a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts index f38ffbf6a..f5bba1150 100644 --- a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts +++ b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts @@ -1,6 +1,7 @@ import { stringifyValue } from "../utils/stringifyValue.ts"; import { getAllSpansInTree } from "../store/helpers.ts"; import type { AILibraryHandler, AIToolCall, AITrace, Span } from "../types.ts"; +import { extractFinishReason } from "./finishReason.ts"; // https://ai-sdk.dev/docs/ai-sdk-core/telemetry const AI_SPAN_OP_PREFIX = "gen_ai."; @@ -78,14 +79,6 @@ const AI_FIELD_MAPPINGS = { ], MODEL_PROVIDER: [AI_MODEL_PROVIDER_FIELD, GEN_AI_SYSTEM_FIELD], FUNCTION_ID: [AI_TELEMETRY_FUNCTION_ID_FIELD, GEN_AI_FUNCTION_ID_FIELD], - PROMPT_TOKENS: [ - AI_USAGE_PROMPT_TOKENS_FIELD, - GEN_AI_USAGE_INPUT_TOKENS_FIELD, - ], - COMPLETION_TOKENS: [ - AI_USAGE_COMPLETION_TOKENS_FIELD, - GEN_AI_USAGE_OUTPUT_TOKENS_FIELD, - ], PROMPT: [GEN_AI_PROMPT_FIELD, AI_PROMPT_FIELD], OPERATION_ID: [ AI_OPERATION_ID_FIELD, @@ -169,6 +162,8 @@ export const vercelAISDKHandler: AILibraryHandler = { rawSpan: rootSpan, metadata: { metadata: {}, + promptTokens: tokenUsage.promptTokens, + completionTokens: tokenUsage.completionTokens, }, toolCalls: [], }; @@ -205,100 +200,77 @@ function extractTokenUsage(spans: Span[]): { promptTokens?: number; completionTokens?: number; } { - let promptTokens: number | undefined; - let completionTokens: number | undefined; - - for (const span of spans) { - if (!span.data) continue; - - if (promptTokens === undefined) { - for (const field of TOKEN_FIELDS.PROMPT) { - const value = span.data[field]; - if (value !== undefined) { - promptTokens = Number(value); + const totals = new Map< + Span, + { promptTokens?: number; completionTokens?: number } + >(); + for (let index = spans.length - 1; index >= 0; index--) { + const span = spans[index]; + const usage: { promptTokens?: number; completionTokens?: number } = {}; + for (const [key, fields] of [ + ["promptTokens", TOKEN_FIELDS.PROMPT], + ["completionTokens", TOKEN_FIELDS.COMPLETION], + ] as const) { + let reported: number | undefined; + for (const field of fields) { + const value = finiteNumber(span.data?.[field]); + if (value !== undefined && value >= 0) { + reported = value; break; } } - } - - if (completionTokens === undefined) { - for (const field of TOKEN_FIELDS.COMPLETION) { - const value = span.data[field]; - if (value !== undefined) { - completionTokens = Number(value); - break; - } + let observed: number | undefined; + for (const child of new Set(span.children)) { + const value = totals.get(child)?.[key]; + if (value !== undefined) observed = (observed ?? 0) + value; } + // SDK versions report either last-step usage or an aggregate on agent + // parents, and Sentry normalizes both to the same keys. Reconcile each + // reported subtotal with its non-overlapping children instead of adding + // parent totals again or replacing a larger total with partial coverage. + usage[key] = + observed === undefined || !Number.isFinite(observed) + ? (reported ?? observed) + : reported === undefined + ? observed + : Math.max(reported, observed); } - - if (promptTokens !== undefined && completionTokens !== undefined) { - break; - } + totals.set(span, usage); } - - return { promptTokens, completionTokens }; + const usage = totals.get(spans[0]); + // An unrepresentable child sum cannot become a public Infinity total. + return { + promptTokens: finiteNumber(usage?.promptTokens), + completionTokens: finiteNumber(usage?.completionTokens), + }; } function determineOperation( spans: Span[], rootSpan: Span, ): { operation: string; hasToolCall: boolean; toolCallName?: string } { - let operation = UNKNOWN_OPERATION; - let hasToolCall = false; - let toolCallName: string | undefined; - let foundToolCallAsOperationId = false; - - for (const span of spans) { - if ( + const operationId = + rootSpan.data && extractFieldFromSpan(rootSpan.data, "OPERATION_ID"); + const operation = + typeof operationId === "string" && operationId + ? operationId + : rootSpan.op || rootSpan.description || UNKNOWN_OPERATION; + const isTool = (span: Span) => { + const id = span.data && extractFieldFromSpan(span.data, "OPERATION_ID"); + return ( span.op === AI_TOOL_CALL_OPERATION || - span.op === AI_EXECUTE_TOOL_OPERATION - ) { - hasToolCall = true; - } - - if (!span.data) continue; - - const operationId = extractFieldFromSpan(span.data, "OPERATION_ID") as - | string - | undefined; - - // Handle tool call operation - if (operationId === AI_TOOL_CALL_OPERATION) { - operation = AI_TOOL_CALL_OPERATION; - foundToolCallAsOperationId = true; - hasToolCall = true; - - if (span.data[GEN_AI_TOOL_NAME_FIELD]) { - toolCallName = stringifyValue(span.data[GEN_AI_TOOL_NAME_FIELD]); - } - } - - // Handle other operations (prioritize root and direct children) - if ( - !foundToolCallAsOperationId && - operationId && - operation === UNKNOWN_OPERATION - ) { - const isHighPriority = - span.span_id === rootSpan.span_id || - rootSpan.children?.some((child) => child.span_id === span.span_id); - - if (isHighPriority) { - operation = operationId; - } - } - } - - // Fallback logic - if (operation === UNKNOWN_OPERATION) { - if (rootSpan.op) { - operation = rootSpan.op; - } else if (rootSpan.description) { - operation = rootSpan.description; - } - } - - return { operation, hasToolCall, toolCallName }; + span.op === AI_EXECUTE_TOOL_OPERATION || + id === AI_TOOL_CALL_OPERATION || + id === AI_EXECUTE_TOOL_OPERATION || + id === "execute_tool" + ); + }; + const name = rootSpan.data?.[GEN_AI_TOOL_NAME_FIELD]; + return { + operation, + hasToolCall: spans.some(isTool), + toolCallName: isTool(rootSpan) && name ? stringifyValue(name) : undefined, + }; } function determineTraceName( @@ -390,36 +362,18 @@ function extractAIMetadata(span: Span, trace: AITrace) { } } - const maxRetries = finiteSetting(span.data[AI_SETTINGS_MAX_RETRIES_FIELD]); + const maxRetries = finiteNumber(span.data[AI_SETTINGS_MAX_RETRIES_FIELD]); if (maxRetries !== undefined) { trace.metadata.maxRetries = maxRetries; } - const maxSteps = finiteSetting(span.data[AI_SETTINGS_MAX_STEPS_FIELD]); + const maxSteps = finiteNumber(span.data[AI_SETTINGS_MAX_STEPS_FIELD]); if (maxSteps !== undefined) { trace.metadata.maxSteps = maxSteps; } - - // Token usage - use === undefined to allow 0 values - if (trace.metadata.promptTokens === undefined) { - const promptTokens = extractFieldFromSpan(span.data, "PROMPT_TOKENS"); - if (promptTokens !== undefined) { - trace.metadata.promptTokens = Number(promptTokens); - } - } - - if (trace.metadata.completionTokens === undefined) { - const completionTokens = extractFieldFromSpan( - span.data, - "COMPLETION_TOKENS", - ); - if (completionTokens !== undefined) { - trace.metadata.completionTokens = Number(completionTokens); - } - } } -function finiteSetting(value: unknown): number | undefined { +function finiteNumber(value: unknown): number | undefined { if ( typeof value !== "number" && (typeof value !== "string" || !value.trim()) @@ -497,13 +451,11 @@ function extractResponseData(span: Span, trace: AITrace) { trace.response.finishReason = stringifyValue(finishReason); } - const finishReasons = span.data[GEN_AI_RESPONSE_FINISH_REASONS_FIELD]; - if ( - !trace.response.finishReason && - Array.isArray(finishReasons) && - finishReasons.length > 0 - ) { - trace.response.finishReason = stringifyValue(finishReasons[0]); + if (!trace.response.finishReason) { + const reason = extractFinishReason( + span.data[GEN_AI_RESPONSE_FINISH_REASONS_FIELD], + ); + if (reason !== undefined) trace.response.finishReason = reason; } const responseText = span.data[AI_RESPONSE_TEXT_FIELD]; diff --git a/packages/spotlight/src/telemetry/data/profiles.test.ts b/packages/spotlight/src/telemetry/data/profiles.test.ts index 1434a6402..13acb45fe 100644 --- a/packages/spotlight/src/telemetry/data/profiles.test.ts +++ b/packages/spotlight/src/telemetry/data/profiles.test.ts @@ -31,7 +31,14 @@ function profile( } describe("profile span frames", () => { - it.each([undefined, "ruby", "javascript"])( + it.each([ + undefined, + "ruby", + "javascript", + "constructor", + "__proto__", + "toString", + ])( "retains the first frame without a matching SDK frame on %s", (platform) => { const spans = getSpansFromProfile( @@ -68,6 +75,47 @@ describe("profile span frames", () => { expect(spans).toMatchObject([{ description: "app", op: "main" }]); }); + it.each([ + "@opentelemetry/instrumentation", + "@opentelemetry/instrumentation-http", + "@opentelemetry.instrumentation.http", + ])("removes OpenTelemetry SDK frames from %s", (module) => { + const spans = getSpansFromProfile( + trace(), + profile({ + platform: "javascript", + frames: [ + { module, function: "instrument" }, + { module: "app", function: "main" }, + ], + stacks: [[0, 1]], + }), + undefined, + 0, + 10, + new Set(), + ); + expect(spans).toMatchObject([{ description: "app", op: "main" }]); + }); + + it.each([ + "@opentelemetry-user/instrumentation-http", + "@opentelemetry/instrumentationhttp", + ])("retains user scopes and lookalike names such as %s", (module) => { + const spans = getSpansFromProfile( + trace(), + profile({ + platform: "javascript", + frames: [{ module, function: "main" }], + }), + undefined, + 0, + 10, + new Set(), + ); + expect(spans).toMatchObject([{ description: module, op: "main" }]); + }); + it.each([ { module: "@angular/core", @@ -226,14 +274,23 @@ describe("profile graft thread selection", () => { "uses the Sentry thread.id from the %s span", (source) => { const target = trace(); + const root: Span = { + span_id: "root", + start_timestamp: 0, + timestamp: 10, + }; + target.spans.set(root.span_id, root); const neighbor: Span = { span_id: "neighbor", + parent_span_id: "root", start_timestamp: source === "next" ? 5 : 0, timestamp: source === "next" ? 10 : 5, data: { "thread.id": 7 }, }; - if (source === "parent") target.data = { "thread.id": 7 }; - else target.spanTree = [neighbor]; + if (source === "parent") root.data = { "thread.id": 7 }; + else { + target.spans.set(neighbor.span_id, neighbor); + } const start = source === "previous" ? 5 : 0; graftProfileSpans( target, @@ -243,7 +300,9 @@ describe("profile graft thread selection", () => { }), ); expect( - target.spanTree.filter((span) => span.tags?.source === "profile"), + target.spans + .get("root") + ?.children?.filter((span) => span.tags?.source === "profile"), ).toMatchObject([{ description: "app", op: "main" }]); expect(target.profileGrafted).toBe(true); }, @@ -251,7 +310,12 @@ describe("profile graft thread selection", () => { it("retains active-thread and legacy threadId fallbacks", () => { const target = trace(); - target.data = { threadId: "worker" }; + target.spans.set("root", { + span_id: "root", + start_timestamp: 0, + timestamp: 10, + data: { threadId: "worker" }, + }); graftProfileSpans( target, profile({ @@ -267,9 +331,8 @@ describe("profile graft thread selection", () => { ], }), ); - expect(target.spanTree.map((span) => span.children?.[0]?.op)).toEqual([ - "main", - "worker", - ]); + expect( + target.spans.get("root")?.children?.map((span) => span.children?.[0]?.op), + ).toEqual(["main", "worker"]); }); }); diff --git a/packages/spotlight/src/telemetry/data/profiles.ts b/packages/spotlight/src/telemetry/data/profiles.ts index 96f3f75e4..0915a2ebb 100644 --- a/packages/spotlight/src/telemetry/data/profiles.ts +++ b/packages/spotlight/src/telemetry/data/profiles.ts @@ -2,7 +2,7 @@ import { log } from "../logger.ts"; import type { SentryProfileWithTraceMeta } from "../store/types.ts"; import type { Span, Trace } from "../types.ts"; import { getFunctionNameFromFrame } from "../utils/profileUtils.ts"; -import { compareSpans } from "../utils/traces.ts"; +import { compareSpans, groupSpans } from "../utils/traces.ts"; /** Non-cryptographic FNV-1a identity for a derived profile frame, stable across replays. */ function profileSpanId(parts: unknown[]): string { @@ -23,12 +23,32 @@ function profileSpanId(parts: unknown[]): string { * @returns Span[] A list of spans with the same description and op consolidated into a single span */ function consolidateSpans(trace: Trace, spans: Span[]): Span[] { - const consolidatedSpans: Span[] = []; - let lastSpan = spans[0]; - let spanIdx = 1; - while (spanIdx < spans.length + 1) { - const span = spans[spanIdx]; - spanIdx += 1; + type Level = { + spans: Span[]; + result: Span[]; + lastSpan?: Span; + index: number; + owner?: Span; + pendingSpan?: Span; + }; + const result: Span[] = []; + const levels: Level[] = [{ spans, result, lastSpan: spans[0], index: 1 }]; + while (levels.length) { + const level = levels[levels.length - 1]; + if (level.pendingSpan) { + // Match recursive postorder: children enter the lookup before their parent. + level.result.push(level.pendingSpan); + trace.spans.set(level.pendingSpan.span_id, level.pendingSpan); + level.pendingSpan = undefined; + continue; + } + if (level.index > level.spans.length) { + if (level.owner) level.owner.children = level.result; + levels.pop(); + continue; + } + const lastSpan = level.lastSpan!; + const span = level.spans[level.index++]; if ( span && span.description === lastSpan.description && @@ -49,19 +69,26 @@ function consolidateSpans(trace: Trace, spans: Span[]): Span[] { } } } else { - // Discard very short spans + level.lastSpan = span; + // Discard very short spans. if (lastSpan.timestamp - lastSpan.start_timestamp > 0) { if (lastSpan.children) { - lastSpan.children = consolidateSpans(trace, lastSpan.children); + level.pendingSpan = lastSpan; + levels.push({ + spans: lastSpan.children, + result: [], + lastSpan: lastSpan.children[0], + index: 1, + owner: lastSpan, + }); + } else { + level.result.push(lastSpan); + trace.spans.set(lastSpan.span_id, lastSpan); } - consolidatedSpans.push(lastSpan); - trace.spans.set(lastSpan.span_id, lastSpan); } - lastSpan = span; } } - - return consolidatedSpans; + return result; } // This is done per platform right now but we may want to make it use @@ -83,7 +110,8 @@ const SENTRY_FRAME_FILTER_PER_PLATFORM: Record< if (module) { return ( module.startsWith("@sentry") || - module.startsWith("@opentelemetry.instrumentation") + module.startsWith("@opentelemetry.instrumentation") || + /^@opentelemetry\/instrumentation(?:$|[-/])/.test(module) ); } // This one below is to match things like `http://localhost:3000/node_modules/.vite/deps/@sentry_react.js?v=6942e78f` etc. @@ -106,7 +134,10 @@ export function getSpansFromProfile( threadIds.add(profile.active_thread_id); const sentryFrameFilter = - profile.platform && SENTRY_FRAME_FILTER_PER_PLATFORM[profile.platform]; + profile.platform && + Object.hasOwn(SENTRY_FRAME_FILTER_PER_PLATFORM, profile.platform) + ? SENTRY_FRAME_FILTER_PER_PLATFORM[profile.platform] + : undefined; const nextSampleTimestamps: Array = []; const nextTimestampByThread = new Map(); for ( @@ -226,64 +257,100 @@ function getThreadId(span: Span | Trace | undefined): string | undefined { } /** - * Modifies the spanTree in place recursively by adding spans from the - * profile data where there are gaps in the trace data. - * @param spanTree Span[] The tree of spans to graft profile spans into + * Rebuilds derived profile spans from the latest samples and real span topology. + * Real spans retain their transaction references; stale derived children and + * missing-parent wrappers are reconstructed rather than grafted a second time. */ export function graftProfileSpans( trace: Trace, profile: SentryProfileWithTraceMeta, +) { + trace.spans = new Map( + Array.from(trace.spans) + .filter( + ([, span]) => span.tags?.source !== "profile" && span.op !== "orphan", + ) + .map(([id, span]) => [id, { ...span, children: undefined }]), + ); + trace.spanTree = groupSpans(trace.spans); + let start = Number.POSITIVE_INFINITY; + let end = Number.NEGATIVE_INFINITY; + for (const span of trace.spans.values()) { + if (span.op === "orphan") continue; + if ( + Number.isFinite(span.start_timestamp) && + Number.isFinite(span.timestamp) + ) { + start = Math.min(start, span.start_timestamp); + end = Math.max(end, span.timestamp); + } + } + // Trace timestamps include context-only errors and logs. Profile gaps must + // instead follow captured work, including real spans beyond a transaction. + if (start < end) { + graftProfileSpansIntoTree(trace, profile, trace.spanTree, { + ...trace, + start_timestamp: start, + timestamp: end, + }); + } + trace.profileGrafted = true; +} + +function graftProfileSpansIntoTree( + trace: Trace, + profile: SentryProfileWithTraceMeta, spanTree: Span[] = trace.spanTree, parent: Span | Trace = trace, ) { log(`Grafting profile spans into trace ${trace.trace_id}`); - if (trace.profileGrafted) { - log(`Trace already has profile grafted ${trace.trace_id}`); - return; - } - - let idx = -1; - while (idx < spanTree.length) { - const span = spanTree[idx] as Span | undefined; + const levels: { spanTree: Span[]; parent: Span | Trace; index: number }[] = [ + { spanTree, parent, index: -1 }, + ]; + while (levels.length) { + const level = levels[levels.length - 1]; + if (level.index >= level.spanTree.length) { + levels.pop(); + continue; + } + const span = level.spanTree[level.index] as Span | undefined; if (span?.tags?.source === "profile") { - idx += 1; + level.index++; continue; } - const nextSpan = spanTree[idx + 1]; + const nextSpan = level.spanTree[level.index + 1]; if (nextSpan?.tags?.source === "profile") { - idx += 1; + level.index++; continue; } - const startTs = span ? span.timestamp : parent.start_timestamp; - const endTs = nextSpan ? nextSpan.start_timestamp : parent.timestamp; + const startTs = span ? span.timestamp : level.parent.start_timestamp; + const endTs = nextSpan ? nextSpan.start_timestamp : level.parent.timestamp; const threadIds = new Set([ getThreadId(span), getThreadId(nextSpan), - getThreadId(parent), + getThreadId(level.parent), ]); threadIds.delete(undefined); if (endTs - startTs > 0) { const fillers = getSpansFromProfile( trace, profile, - parent.span_id, + level.parent.span_id, startTs, endTs, threadIds as Set, ); if (fillers.length) { - spanTree.splice(idx + 1, 0, ...fillers); - idx += fillers.length; + level.spanTree.splice(level.index + 1, 0, ...fillers); + level.index += fillers.length; } } + level.index++; if (span) { span.children ??= []; - graftProfileSpans(trace, profile, span.children, span); + // Resume the current sibling level only after its child's gaps are grafted. + levels.push({ spanTree: span.children, parent: span, index: -1 }); } - idx += 1; } - // Only mark as grafted at the top level to avoid early quitting during - // recursive calls above for child spans - trace.profileGrafted = trace.spanTree === spanTree; } diff --git a/packages/spotlight/src/telemetry/profile-depth.spec.ts b/packages/spotlight/src/telemetry/profile-depth.spec.ts new file mode 100644 index 000000000..73d406dda --- /dev/null +++ b/packages/spotlight/src/telemetry/profile-depth.spec.ts @@ -0,0 +1,115 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "../sdk/core.ts"; + +const depth = 12000; + +function transaction(spans: unknown[] = []) { + return [ + { type: "transaction" }, + { + event_id: "transaction", + type: "transaction", + transaction: "deep profile", + start_timestamp: 1, + timestamp: 2, + contexts: { trace: { trace_id: "trace", span_id: "root" } }, + spans, + }, + ]; +} + +function profile(frameCount: number) { + return [ + { type: "profile" }, + { + event_id: "profile", + version: "1", + timestamp: 1, + platform: "javascript", + transaction: { trace_id: "trace", active_thread_id: "main" }, + profile: { + frames: Array.from({ length: frameCount }, (_, index) => ({ + module: "app", + function: `frame-${index}`, + })), + stacks: [ + Array.from( + { length: frameCount }, + (_, index) => frameCount - index - 1, + ), + ], + samples: [ + { + thread_id: "main", + stack_id: 0, + elapsed_since_start_ns: "100000000", + }, + { + thread_id: "main", + stack_id: 0, + elapsed_since_start_ns: "200000000", + }, + ], + }, + }, + ]; +} + +test("public ingestion retains and deterministically consolidates a 12000-frame profile stack", async () => { + const store = createTelemetryStore(); + try { + const input = [{}, [transaction(), profile(depth)]]; + expect((await store.ingestEnvelope(input)).status).toBe("processed"); + expect(store.getProfile("trace")?.frames).toHaveLength(depth); + const trace = store.getTrace("trace")!; + const derived = [...trace.spans.values()].filter( + (span) => span.tags?.source === "profile", + ); + expect(derived.filter((span) => span.op !== "Thread")).toHaveLength(depth); + let current = trace.spans.get("root")?.children?.[0]; + for (let index = 0; index < depth; index++) { + expect(current?.op).toBe(`frame-${index}`); + expect(current?.start_timestamp).toBe(1100); + expect(current?.timestamp).toBe(2000); + current = current?.children?.[0]; + } + expect(current).toBeUndefined(); + const identities = [...trace.spans.keys()]; + store.reset(); + expect((await store.ingestEnvelope(input)).status).toBe("processed"); + expect([...store.getTrace("trace")!.spans.keys()]).toEqual(identities); + } finally { + store.dispose(); + } +}); + +test("public ingestion grafts profile frames below a 12000-span real chain", async () => { + const store = createTelemetryStore(); + try { + const spans = Array.from({ length: depth }, (_, index) => ({ + span_id: `real-${index}`, + parent_span_id: index === 0 ? "root" : `real-${index - 1}`, + op: "function", + description: `real-${index}`, + start_timestamp: 1, + timestamp: 2, + })); + expect( + (await store.ingestEnvelope([{}, [transaction(spans), profile(1)]])) + .status, + ).toBe("processed"); + expect(store.getProfile("trace")?.frames).toHaveLength(1); + const trace = store.getTrace("trace")!; + expect(trace.spans.get(`real-${depth - 1}`)?.children).toMatchObject([ + { op: "frame-0", start_timestamp: 1100, timestamp: 2000 }, + ]); + expect( + [...trace.spans.values()].filter( + (span) => span.tags?.source !== "profile", + ), + ).toHaveLength(depth + 1); + expect(trace.spans.get("root")?.transaction).toBe(trace.transactions[0]); + } finally { + store.dispose(); + } +}); diff --git a/packages/spotlight/src/telemetry/profiles-streaming.spec.ts b/packages/spotlight/src/telemetry/profiles-streaming.spec.ts new file mode 100644 index 000000000..cbcd05510 --- /dev/null +++ b/packages/spotlight/src/telemetry/profiles-streaming.spec.ts @@ -0,0 +1,429 @@ +import { readFileSync } from "node:fs"; +import { expect, test } from "vitest"; +import { serializeTrace } from "../sdk/core.ts"; +import { createTelemetryPipeline } from "../sdk/node.ts"; + +test("later continuous chunks refresh the public profile, summaries and grafted spans", async () => { + const pipeline = createTelemetryPipeline(); + try { + for (const file of ["transaction.txt", "profile_chunk_1.txt"]) { + const result = await pipeline.ingestPayload({ + body: readFileSync( + new URL( + `../../_fixtures/continuous_profiling/${file}`, + import.meta.url, + ), + ), + }); + expect(result.status).toBe("processed"); + } + const traceId = "f1e2d3c4b5a6978012345678901234ab"; + expect(pipeline.getProfile(traceId)?.samples).toHaveLength(7); + await pipeline.ingestEnvelope([ + {}, + [ + [ + { type: "profile_chunk" }, + { + event_id: "later-chunk", + type: "profile_chunk", + version: "2", + profiler_id: "71bba98d90b545c39f2ae73f702d7ef4", + chunk_id: "later-chunk", + platform: "cocoa", + profile: { + frames: [{ module: "App", function: "laterWork" }], + stacks: [[0]], + samples: [ + { timestamp: 1724770811.573, stack_id: 0, thread_id: "259" }, + { timestamp: 1724770811.583, stack_id: 0, thread_id: "259" }, + ], + }, + }, + ], + ], + ]); + expect(pipeline.getProfile(traceId)?.samples).toHaveLength(9); + expect(pipeline.getProfileFunctionSummaries({ traceId })).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "App@laterWork", sampleCount: 1 }), + ]), + ); + expect([...pipeline.getTrace(traceId)!.spans.values()]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + op: "laterWork", + tags: { source: "profile" }, + }), + ]), + ); + } finally { + pipeline.dispose(); + } +}); + +function transaction(eventId: string, start = 1, end = 1.4) { + return [ + { type: "transaction" }, + { + event_id: eventId, + type: "transaction", + platform: "javascript", + transaction: "streamed profile", + start_timestamp: start, + timestamp: end, + contexts: { + trace: { + trace_id: "trace", + span_id: "root", + data: { "thread.id": "main" }, + }, + profile: { profiler_id: "profiler" }, + }, + spans: [ + { + span_id: "embedded", + parent_span_id: "root", + description: "real work", + start_timestamp: 1.15, + timestamp: 1.18, + }, + ], + }, + ]; +} + +function chunk(id: string, fn: string, timestamps: number[]) { + return [ + { type: "profile_chunk" }, + { + event_id: id, + type: "profile_chunk", + version: "2", + profiler_id: "profiler", + chunk_id: id, + platform: "javascript", + profile: { + frames: [{ module: "app", function: fn }], + stacks: [[0]], + samples: timestamps.map((timestamp) => ({ + timestamp, + stack_id: 0, + thread_id: "main", + })), + }, + }, + ]; +} + +const streamed = [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "trace", + span_id: "streamed", + parent_span_id: "root", + name: "streamed work", + start_timestamp: 1.28, + end_timestamp: 1.29, + }, + ], + }, +]; +const javascriptHeader = { + sdk: { name: "sentry.javascript.node", version: "10.63.0" }, +}; + +test("profile chunks, late timing and streamed spans converge across reordered batches without stale derived spans", async () => { + const inputs = [ + transaction("initial"), + chunk("first", "first", [1, 1.1]), + chunk("second", "second", [1.2, 1.3]), + streamed, + transaction("timing", 0.95, 1.5), + ]; + const baseline = createTelemetryPipeline(); + const reordered = createTelemetryPipeline(); + try { + expect( + (await baseline.ingestEnvelope([javascriptHeader, inputs])).status, + ).toBe("processed"); + const expectedProfile = baseline.getProfile("trace"); + const expectedSummary = baseline.getProfileFunctionSummaries(); + const expectedTrace = serializeTrace(baseline.getTrace("trace")!); + expectedTrace.spans.sort(([a], [b]) => a.localeCompare(b)); + for (const order of [ + [1, 2, 0, 3, 4], + [2, 0, 3, 1, 4], + [0, 1, 4, 3, 2], + ]) { + reordered.reset(); + for (const index of order) { + expect( + (await reordered.ingestEnvelope([javascriptHeader, [inputs[index]]])) + .status, + ).toBe("processed"); + } + expect(reordered.getProfile("trace")).toEqual(expectedProfile); + expect(reordered.getProfileFunctionSummaries()).toEqual(expectedSummary); + const trace = reordered.getTrace("trace")!; + const serialized = serializeTrace(trace); + serialized.spans.sort(([a], [b]) => a.localeCompare(b)); + expect(serialized).toEqual(expectedTrace); + expect( + [...trace.spans.values()] + .filter((span) => span.tags?.source !== "profile") + .map((span) => span.span_id) + .sort(), + ).toEqual(["embedded", "root", "streamed"]); + expect(trace.spans.get("root")?.transaction).toBe(trace.transactions[0]); + const beforeReplay = serializeTrace(trace); + expect( + (await reordered.ingestEnvelope([javascriptHeader, [inputs[2]]])) + .status, + ).toBe("processed"); + expect(serializeTrace(reordered.getTrace("trace")!)).toEqual( + beforeReplay, + ); + } + } finally { + baseline.dispose(); + reordered.dispose(); + } +}); + +test("appending chunks preserves IDs for existing derived nodes and earlier snapshots", async () => { + const pipeline = createTelemetryPipeline(); + try { + await pipeline.ingestEnvelope([ + javascriptHeader, + [transaction("initial"), chunk("first", "first", [1, 1.1])], + ]); + const earlier = pipeline.getTrace("trace")!; + const previousIds = [...earlier.spans.values()] + .filter((span) => span.op === "first" && span.start_timestamp === 1000) + .map((span) => span.span_id); + expect(previousIds).toHaveLength(1); + const serializedEarlier = serializeTrace(earlier); + await pipeline.ingestEnvelope([ + javascriptHeader, + [chunk("second", "second", [1.2, 1.3])], + ]); + const latest = pipeline.getTrace("trace")!; + expect( + [...latest.spans.values()] + .filter((span) => span.op === "first" && span.start_timestamp === 1000) + .map((span) => span.span_id), + ).toEqual(previousIds); + expect( + [...latest.spans.values()].some((span) => span.op === "second"), + ).toBe(true); + expect(serializeTrace(earlier)).toEqual(serializedEarlier); + } finally { + pipeline.dispose(); + } +}); + +test("context-only errors do not extend an existing profile graft", async () => { + const pipeline = createTelemetryPipeline(); + try { + await pipeline.ingestEnvelope([ + javascriptHeader, + [transaction("initial"), chunk("first", "first", [1, 1.1])], + ]); + const before = [...pipeline.getTrace("trace")!.spans.values()]; + const profileBefore = pipeline.getProfile("trace"); + await pipeline.ingestEnvelope([ + javascriptHeader, + [ + [ + { type: "event" }, + { + event_id: "later-error", + timestamp: 9, + contexts: { trace: { trace_id: "trace" } }, + exception: { values: [{ type: "Error", value: "context only" }] }, + }, + ], + ], + ]); + expect([...pipeline.getTrace("trace")!.spans.values()]).toEqual(before); + expect(pipeline.getProfile("trace")).toEqual(profileBefore); + } finally { + pipeline.dispose(); + } +}); + +test("profile refresh uses real topology bounds across error and chunk ordering", async () => { + const inputs = [ + transaction("initial"), + chunk("first", "first", [1, 1.1]), + [ + { type: "event" }, + { + event_id: "later-error", + timestamp: 9, + contexts: { trace: { trace_id: "trace" } }, + exception: { values: [{ type: "Error", value: "context only" }] }, + }, + ], + chunk("later", "later", [1.5, 1.6]), + ]; + let expectedSpans: ReturnType["spans"] | undefined; + for (const order of [ + [0, 1, 2, 3], + [0, 1, 3, 2], + [2, 0, 1, 3], + ]) { + const pipeline = createTelemetryPipeline(); + try { + for (const index of order) + await pipeline.ingestEnvelope([javascriptHeader, [inputs[index]]]); + const trace = pipeline.getTrace("trace")!; + expect(trace.timestamp).toBe(9000); + const derived = [...trace.spans.values()].filter( + (span) => span.tags?.source === "profile", + ); + expect(derived.some((span) => span.op === "first")).toBe(true); + expect(derived.some((span) => span.op === "later")).toBe(false); + expect( + derived.every( + (span) => span.start_timestamp >= 1000 && span.timestamp <= 1400, + ), + ).toBe(true); + const serialized = serializeTrace(trace).spans.sort(([a], [b]) => + a.localeCompare(b), + ); + if (expectedSpans) expect(serialized).toEqual(expectedSpans); + else expectedSpans = serialized; + + await pipeline.ingestEnvelope([ + javascriptHeader, + [ + [ + { type: "span" }, + { + trace_id: "trace", + span_id: "late-real", + op: "function", + start_timestamp: 1.7, + timestamp: 1.8, + }, + ], + ], + ]); + const extended = pipeline.getTrace("trace")!; + const refreshed = [...extended.spans.values()].filter( + (span) => span.tags?.source === "profile", + ); + expect(extended.timestamp).toBe(9000); + expect(refreshed.some((span) => span.op === "later")).toBe(true); + expect(refreshed.every((span) => span.timestamp <= 1800)).toBe(true); + expect(extended.spans.get("root")?.transaction).toBe( + extended.transactions[0], + ); + } finally { + pipeline.dispose(); + } + } +}); + +test("a linked profile remains readable without grafting context-only trace time", async () => { + const pipeline = createTelemetryPipeline(); + try { + await pipeline.ingestEnvelope([ + javascriptHeader, + [ + [ + { type: "event" }, + { + event_id: "early", + timestamp: 1, + contexts: { trace: { trace_id: "trace" } }, + }, + ], + [ + { type: "event" }, + { + event_id: "late", + timestamp: 9, + contexts: { trace: { trace_id: "trace" } }, + }, + ], + [ + { type: "profile" }, + { + event_id: "profile-only", + type: "profile", + version: "1", + timestamp: 1, + platform: "javascript", + transaction: { trace_id: "trace", active_thread_id: "main" }, + profile: { + frames: [{ module: "app", function: "work" }], + stacks: [[0]], + samples: [ + { thread_id: "main", stack_id: 0, elapsed_since_start_ns: "0" }, + { + thread_id: "main", + stack_id: 0, + elapsed_since_start_ns: "100000000", + }, + ], + }, + }, + ], + ], + ]); + expect(pipeline.getTrace("trace")?.timestamp).toBe(9000); + expect(pipeline.getProfile("trace")?.samples).toHaveLength(2); + expect(pipeline.getTrace("trace")?.spans.size).toBe(0); + expect(pipeline.getTrace("trace")?.spanTree).toEqual([]); + } finally { + pipeline.dispose(); + } +}); + +test.each([{}, { sdk: { name: "sentry.python", version: "2.0.0" } }])( + "explicit chunk platform survives batching with an optional or conflicting SDK header %j", + async (header) => { + const combined = createTelemetryPipeline(); + const split = createTelemetryPipeline(); + try { + const txn = transaction("initial"); + delete (txn[1] as { platform?: string }).platform; + const profileChunk = chunk("first", "first", [1, 1.1]); + await combined.ingestEnvelope([header, [txn, profileChunk]]); + await split.ingestEnvelope([header, [txn]]); + await split.ingestEnvelope([header, [profileChunk]]); + expect(combined.getProfile("trace")?.platform).toBe("javascript"); + expect(split.getProfile("trace")).toEqual(combined.getProfile("trace")); + expect(combined.getEvent("first")?.platform).toBe("javascript"); + expect(split.getEvent("first")?.platform).toBe("javascript"); + } finally { + combined.dispose(); + split.dispose(); + } + }, +); + +test.each([undefined, null, 17, "", " "])( + "infers a missing or invalid chunk platform %j from the envelope SDK", + async (platform) => { + const pipeline = createTelemetryPipeline(); + try { + const profileChunk = chunk("first", "first", [1, 1.1]); + (profileChunk[1] as { platform?: unknown }).platform = platform; + await pipeline.ingestEnvelope([ + javascriptHeader, + [transaction("initial"), profileChunk], + ]); + expect(pipeline.getProfile("trace")?.platform).toBe("javascript"); + expect(pipeline.getEvent("first")?.platform).toBe("javascript"); + } finally { + pipeline.dispose(); + } + }, +); diff --git a/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts b/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts index 32a258965..8f4b09390 100644 --- a/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts +++ b/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts @@ -136,7 +136,10 @@ export const createEnvelopesSlice = if (SUPPORTED_EVENT_TYPES.has(itemHeader.type)) { const item = structuredClone(itemData) as SentryEvent; item.type ??= itemHeader.type as SentryEvent["type"]; - if (!RAW_TYPES.has(itemHeader.type)) { + if ( + !RAW_TYPES.has(itemHeader.type) && + (typeof item.platform !== "string" || !item.platform.trim()) + ) { item.platform = sdkToPlatform(sdk.name); } if (traceContext) { diff --git a/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts index acf9069db..adf67ede2 100644 --- a/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts +++ b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts @@ -1,8 +1,7 @@ import type { StateCreator } from "zustand/vanilla"; import { graftProfileSpans } from "../../data/profiles.ts"; import type { EngineOptions, ProcessingRuntime } from "../../internal/types.ts"; -import type { SentryEvent, SentryLogEventItem } from "../../types.ts"; -import type { SentryTransactionEvent } from "../../types.ts"; +import type { SentryEvent, SentryLogEventItem, Trace } from "../../types.ts"; import { isErrorEvent, isLogEvent, @@ -31,6 +30,32 @@ import { updateTraceMetadata, } from "../utils/traceProcessor.ts"; +function continuousProfileForTrace( + trace: Trace, + chunksByProfilerId: SentryStore["profileChunksByProfilerId"], + profilerId?: string, +): SentryProfileWithTraceMeta | undefined { + for (const transaction of trace.transactions) { + const profileContext = transaction.contexts.profile as + | { profiler_id?: string } + | undefined; + const id = profileContext?.profiler_id; + if (!id || (profilerId !== undefined && profilerId !== id)) continue; + const chunks = chunksByProfilerId.get(id); + if (!chunks?.length) continue; + const threadId = transaction.contexts.trace.data?.["thread.id"]; + return ( + mergeChunksToProfile( + chunks, + typeof threadId === "string" || typeof threadId === "number" + ? String(threadId) + : undefined, + ) ?? undefined + ); + } + return undefined; +} + export const createEventsSlice = ( options: EngineOptions, @@ -203,6 +228,17 @@ export const createEventsSlice = trace.errors += 1; } + // Only real span changes can invalidate the graft's topology or + // timing. Context-only errors/logs must not extend profile samples. + if (isTraceEvent(event) || event.type === "span") { + mergedV2Profile = + continuousProfileForTrace(trace, profileChunksByProfilerId) ?? + mergedV2Profile; + const profile = + mergedV2Profile ?? profilesByTraceId.get(trace.trace_id); + if (profile) graftProfileSpans(trace, profile); + } + // Update trace metadata and status for all events with trace context updateTraceMetadata(trace); @@ -242,12 +278,8 @@ export const createEventsSlice = // Handle V2 profile chunks (continuous profiling) if (isProfileChunkEvent(event)) { - const { - profileChunksByProfilerId, - tracesById, - eventsById, - profilesByTraceId, - } = get(); + const { profileChunksByProfilerId, tracesById, profilesByTraceId } = + get(); const { chunk } = processProfileChunkEvent(event); // Update store with processed profile chunk @@ -265,59 +297,27 @@ export const createEventsSlice = updatedChunks, ); - // Check if there's an existing transaction with this profiler_id that needs re-grafting - // This handles the case where profile chunks arrive AFTER the transaction - let traceToUpdate: string | undefined; - let activeThreadId: string | undefined; - for (const evt of eventsById.values()) { - if (isTraceEvent(evt)) { - const txn = evt as SentryTransactionEvent; - const profileCtx = txn.contexts?.profile as - | { profiler_id?: string } - | undefined; - if (profileCtx?.profiler_id === chunk.profiler_id) { - traceToUpdate = txn.contexts.trace.trace_id; - activeThreadId = txn.contexts.trace.data?.["thread.id"] as - | string - | undefined; - break; - } - } - } - - if (traceToUpdate) { - // Re-graft with updated chunks - const existingTrace = tracesById.get(traceToUpdate); - if (existingTrace && !existingTrace.profileGrafted) { - const mergedProfile = mergeChunksToProfile( - updatedChunks, - activeThreadId, - ); - if (mergedProfile) { - // Create a shallow copy of the trace to trigger React re-render - const trace = { - ...existingTrace, - spans: new Map(existingTrace.spans), - spanTree: [...existingTrace.spanTree], - }; - graftProfileSpans(trace, mergedProfile); - - // Store updated trace and profile - const newTracesById = new Map(tracesById); - newTracesById.set(traceToUpdate, trace); - const newProfilesByTraceId = new Map(profilesByTraceId); - newProfilesByTraceId.set(traceToUpdate, mergedProfile); - set({ - profileChunksByProfilerId: newProfileChunksByProfilerId, - tracesById: newTracesById, - profilesByTraceId: newProfilesByTraceId, - }); - return; - } - } + // All transactions and streamed segments linked to this profiler + // must see every new chunk, even after their first successful graft. + const newTracesById = new Map(tracesById); + const newProfilesByTraceId = new Map(profilesByTraceId); + for (const existingTrace of tracesById.values()) { + const mergedProfile = continuousProfileForTrace( + existingTrace, + newProfileChunksByProfilerId, + chunk.profiler_id, + ); + if (!mergedProfile) continue; + const trace = { ...existingTrace }; + graftProfileSpans(trace, mergedProfile); + newTracesById.set(trace.trace_id, trace); + newProfilesByTraceId.set(trace.trace_id, mergedProfile); } - - set({ profileChunksByProfilerId: newProfileChunksByProfilerId }); + set({ + profileChunksByProfilerId: newProfileChunksByProfilerId, + tracesById: newTracesById, + profilesByTraceId: newProfilesByTraceId, + }); } } } finally { From 2b4e93e6a5b93e9d8c881bc06970de0f8c76f0c4 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 10:18:37 +0530 Subject: [PATCH 04/15] fix(spotlight): Isolate staged cleanup across resets Guard staged-byte cleanup by generation so cancelled parsed work cannot remove a newly admitted delivery with the same ID after reset. Co-Authored-By: Codex --- .../src/sdk/node-history-retention.spec.ts | 61 +++++++++++++++++++ packages/spotlight/src/sdk/node.ts | 8 ++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/packages/spotlight/src/sdk/node-history-retention.spec.ts b/packages/spotlight/src/sdk/node-history-retention.spec.ts index fd7697e1d..f320692df 100644 --- a/packages/spotlight/src/sdk/node-history-retention.spec.ts +++ b/packages/spotlight/src/sdk/node-history-retention.spec.ts @@ -5,6 +5,67 @@ function body(eventId: string): string { return `{}\n{"type":"event"}\n${JSON.stringify({ event_id: eventId, timestamp: 1 })}`; } +test("a cancelled parsed delivery cannot clear a new generation's staged bytes", async () => { + let startOld!: () => void; + let startNew!: () => void; + let releaseOld!: () => void; + let releaseNew!: () => void; + const oldStarted = new Promise((resolve) => { + startOld = resolve; + }); + const newStarted = new Promise((resolve) => { + startNew = resolve; + }); + const oldGate = new Promise((resolve) => { + releaseOld = resolve; + }); + const newGate = new Promise((resolve) => { + releaseNew = resolve; + }); + const pipeline = createTelemetryPipeline({ + enrichStacktrace: async (event) => { + if (event.event_id === "old") { + startOld(); + await oldGate; + } else { + startNew(); + await newGate; + } + }, + }); + const payload = + '{}\n{"type":"event"}\n{"event_id":"new","timestamp":1,"exception":{"values":[]}}'; + try { + const old = pipeline.ingestEnvelope( + [ + {}, + [ + [ + { type: "event" }, + { event_id: "old", timestamp: 1, exception: { values: [] } }, + ], + ], + ], + { deliveryId: "reused" }, + ); + await oldStarted; + pipeline.reset(); + const newest = pipeline.ingestPayload({ + body: payload, + deliveryId: "reused", + }); + await newStarted; + expect(await old).toMatchObject({ status: "ignored", reason: "reset" }); + releaseNew(); + expect(await newest).toMatchObject({ status: "processed" }); + expect(pipeline.getDecodedPayload("reused")?.body.toString()).toBe(payload); + } finally { + releaseOld(); + releaseNew(); + pipeline.dispose(); + } +}); + test.each(["oversized", "within-budget"] as const)( "reusing an evicted delivery ID replaces decoded history: %s", async (scenario) => { diff --git a/packages/spotlight/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts index 94251b3d1..d81856033 100644 --- a/packages/spotlight/src/sdk/node.ts +++ b/packages/spotlight/src/sdk/node.ts @@ -296,7 +296,10 @@ export function createTelemetryPipeline( } return result; } finally { - if (stagedPayloads.get(envelopeId) === payload) + if ( + generation === currentGeneration && + stagedPayloads.get(envelopeId) === payload + ) stagedPayloads.delete(envelopeId); } } @@ -379,7 +382,8 @@ export function createTelemetryPipeline( telemetry.ingestEnvelope(prepared.envelope, prepared.options), ); } finally { - stagedPayloads.delete(envelopeId); + if (epoch === generation && stagedPayloads.get(envelopeId) === null) + stagedPayloads.delete(envelopeId); } } From c01408b1a0e235e338caf4c3075d5ee94eea8f96 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 11:26:26 +0530 Subject: [PATCH 05/15] fix(spotlight): Harden processing and cold test startup Return 415 for unsupported envelope media types, preserve usable AI prompts and tool calls, and clip profile intervals to the available gap. Build the shared package for direct CLI test entry points even with cached schemas. Co-Authored-By: Codex --- packages/cli/package.json | 6 +- .../src/commands/local/telemetry-buffer.ts | 9 +- .../commands/local/telemetry-buffer.test.ts | 57 ++++ packages/spotlight/README.md | 2 +- .../src/telemetry/ai-wire-values.spec.ts | 304 ++++++++++++++++++ packages/spotlight/src/telemetry/ai/genAI.ts | 9 +- .../spotlight/src/telemetry/ai/vercelAISDK.ts | 100 +++--- .../src/telemetry/data/profiles.test.ts | 7 +- .../spotlight/src/telemetry/data/profiles.ts | 6 +- .../src/telemetry/profile-intervals.spec.ts | 91 ++++++ 10 files changed, 534 insertions(+), 57 deletions(-) create mode 100644 packages/spotlight/src/telemetry/ai-wire-values.spec.ts create mode 100644 packages/spotlight/src/telemetry/profile-intervals.spec.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index c6699a75f..3dc3b9bcd 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -65,10 +65,10 @@ "lint": "pnpm -w exec oxlint --report-unused-disable-directives packages/cli && pnpm -w exec oxfmt --check packages/cli", "lint:fix": "pnpm -w exec oxlint --fix packages/cli && pnpm -w exec oxfmt --write packages/cli", "test": "pnpm run test:unit", - "test:unit": "pnpm run generate:docs && pnpm run generate:sdk && vitest run test/lib test/commands test/types test/script --coverage", - "test:changed": "pnpm run generate:docs && pnpm run generate:sdk && vitest run --changed", + "test:unit": "pnpm --filter @sentry/spotlight run build && pnpm run generate:docs && pnpm run generate:sdk && vitest run test/lib test/commands test/types test/script --coverage", + "test:changed": "pnpm --filter @sentry/spotlight run build && pnpm run generate:docs && pnpm run generate:sdk && vitest run --changed", "test:e2e": "pnpm tsx script/prepare-e2e-bundle.ts && vitest run test/e2e", - "test:init-eval": "vitest run test/init-eval --testTimeout 600000", + "test:init-eval": "pnpm --filter @sentry/spotlight run build && vitest run test/init-eval --testTimeout 600000", "generate:sdk": "pnpm tsx script/generate-sdk.ts", "generate:skill": "pnpm tsx script/generate-skill.ts", "generate:banner": "pnpm tsx script/generate-banner-sixel.ts", diff --git a/packages/cli/src/commands/local/telemetry-buffer.ts b/packages/cli/src/commands/local/telemetry-buffer.ts index a49acc7b7..100fbe174 100644 --- a/packages/cli/src/commands/local/telemetry-buffer.ts +++ b/packages/cli/src/commands/local/telemetry-buffer.ts @@ -52,8 +52,15 @@ export function localEnvelopeContentType(value?: string): string { } /** Convert SDK cancellation/admission outcomes into transport retry responses. */ -export function localIngestStatus(receipt: IngestResult): 204 | 400 | 503 { +export function localIngestStatus( + receipt: IngestResult, +): 204 | 400 | 415 | 503 { if (receipt.status === "invalid") return 400; + if ( + receipt.status === "ignored" && + receipt.reason === "unsupported_content_type" + ) + return 415; if ( receipt.status === "ignored" && ["queue_full", "disposed", "reset", "aborted"].includes(receipt.reason) diff --git a/packages/cli/test/commands/local/telemetry-buffer.test.ts b/packages/cli/test/commands/local/telemetry-buffer.test.ts index 04985d6e8..16da1962e 100644 --- a/packages/cli/test/commands/local/telemetry-buffer.test.ts +++ b/packages/cli/test/commands/local/telemetry-buffer.test.ts @@ -13,6 +13,63 @@ const event = (id: string) => `{"event_id":"${id}"}\n{"type":"event"}\n{"message":"hello"}`; describe("shared telemetry receiver", () => { + test.each([ + "stream", + "envelope", + "envelope-slash", + "scoped-ingest", + "scoped-stream", + ])( + "%s rejects unsupported media types without acknowledging or retaining their body", + async (route) => { + const plane = createLocalControlPlane({ controlToken: "media-type" }); + const session = plane.sessionStore.create({ + label: "media-type", + cwd: "/test", + }); + const scoped = route.startsWith("scoped-"); + const buffer = scoped + ? plane.sessionStore.getBuffer(session.id) + : createLocalTelemetryBuffer(); + const app = scoped ? plane.app : buildApp(buffer); + const capabilities = plane.sessionStore.getCapabilities(session.id); + const paths: Record = { + stream: "/stream", + envelope: "/api/42/envelope", + "envelope-slash": "/api/42/envelope/", + "scoped-ingest": `/_local/v1/sessions/${session.id}/ingest/${capabilities.ingest}`, + "scoped-stream": `/_local/v1/sessions/${session.id}/stream?cap=${capabilities.ingest}`, + }; + try { + for (const contentType of [ + "application/json", + "application/json; charset=utf-8", + "application/octet-stream", + ]) { + const response = await app.request(paths[route], { + method: "POST", + headers: { "Content-Type": contentType }, + body: event("rejected"), + }); + expect(response.status).toBe(415); + expect(response.headers.get("Retry-After")).toBeNull(); + expect(buffer.read({ all: true })).toEqual([]); + expect(buffer.pipeline.getEnvelopes()).toEqual([]); + } + const accepted = await app.request(paths[route], { + method: "POST", + headers: { "Content-Type": SENTRY_CONTENT_TYPE }, + body: event("accepted"), + }); + expect(accepted.status).toBe(204); + expect(buffer.read({ all: true })).toHaveLength(1); + } finally { + buffer.dispose(); + if (!scoped) plane.sessionStore.getBuffer(session.id).dispose(); + } + }, + ); + test("accepts streamed span v2 through the shared pipeline", async () => { const buffer = createLocalTelemetryBuffer(); try { diff --git a/packages/spotlight/README.md b/packages/spotlight/README.md index f9eab2cc2..c9499813f 100644 --- a/packages/spotlight/README.md +++ b/packages/spotlight/README.md @@ -81,7 +81,7 @@ An optional `enrichStacktrace(event, { signal })` hook receives a copied event. ### Read APIs and JSON -Accept a `TelemetryReader` from `/sdk/core` in consumers that only query telemetry. Both factories implement it. `createTelemetryReader(getSnapshot)` also constructs a reader over an existing snapshot source. Singular getters such as `getTrace(id)` and `getEvent(id)` return `undefined` when absent; collection getters return readonly arrays. All processed timestamps and durations use milliseconds; time bounds are inclusive Unix milliseconds. +Accept a `TelemetryReader` from `@sentry/spotlight` in consumers that only query telemetry. Both factories implement it. `createTelemetryReader(getSnapshot)` also constructs a reader over an existing snapshot source. Singular getters such as `getTrace(id)` and `getEvent(id)` return `undefined` when absent; collection getters return readonly arrays. All processed timestamps and durations use milliseconds; time bounds are inclusive Unix milliseconds. ```ts const queries = reader.getDatabaseQuerySummaries({ diff --git a/packages/spotlight/src/telemetry/ai-wire-values.spec.ts b/packages/spotlight/src/telemetry/ai-wire-values.spec.ts new file mode 100644 index 000000000..5985257b8 --- /dev/null +++ b/packages/spotlight/src/telemetry/ai-wire-values.spec.ts @@ -0,0 +1,304 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "../sdk/core.ts"; + +function span( + id: string, + data: Record, + parent = "transaction", +) { + return { + trace_id: "ai-wire-trace", + span_id: id, + parent_span_id: parent, + op: "gen_ai.chat", + start_timestamp: 1, + timestamp: 2, + data, + }; +} + +async function queryTrace(spans: ReturnType[]) { + const store = createTelemetryStore(); + try { + const result = await store.ingestEnvelope([ + {}, + [ + [ + { type: "transaction" }, + { + type: "transaction", + event_id: "ai-wire-event", + transaction: "AI request", + start_timestamp: 1, + timestamp: 2, + contexts: { + trace: { + trace_id: "ai-wire-trace", + span_id: "transaction", + op: "http.server", + }, + }, + spans, + }, + ], + ], + ]); + expect(result.status).toBe("processed"); + const traces = store.getAITraces(); + expect(traces).toHaveLength(1); + return traces[0]; + } finally { + store.dispose(); + } +} + +const calls = [ + { toolCallId: "weather-call", toolName: "weather", args: { city: "Pune" } }, +]; + +test.each(["null", "{}", "not JSON"])( + "Vercel AI queries preserve a same-span wrapper after malformed request messages: %s", + async (invalid) => { + for (const wrapper of [ + { + messages: [{ role: "user", content: "Usable conversation" }], + system: "Be concise", + }, + { prompt: "Usable conversation", system: "Be concise" }, + ]) { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.prompt": JSON.stringify(wrapper), + "gen_ai.request.messages": invalid, + }), + ]); + expect(trace?.prompt).toEqual({ + messages: [{ role: "user", content: "Usable conversation" }], + system: "Be concise", + }); + } + }, +); + +test.each(["null", "{}", "not JSON"])( + "Vercel AI queries allow a valid conversation after malformed request messages: %s", + async (invalid) => { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.prompt": JSON.stringify({ + prompt: "Fallback", + system: "Be concise", + }), + }), + span("invalid", { "gen_ai.request.messages": invalid }, "root"), + span( + "valid", + { + "gen_ai.request.messages": JSON.stringify([ + { role: "user", content: "Structured" }, + ]), + }, + "root", + ), + ]); + expect(trace?.prompt).toEqual({ + messages: [{ role: "user", content: "Structured" }], + system: "Be concise", + }); + }, +); +const invalidCalls = ["null", "{}", "3", '"text"', "true", "[", 3, {}, null]; + +test.each(invalidCalls.map((invalid) => ({ invalid })))( + "GenAI queries omit non-array response tool calls: $invalid", + async ({ invalid }) => { + const trace = await queryTrace([ + span("root", { "gen_ai.response.tool_calls": invalid }), + ]); + expect(trace?.response?.toolCalls).toBeUndefined(); + }, +); + +test.each(invalidCalls.map((invalid) => ({ invalid })))( + "GenAI queries preserve valid response tool calls after a malformed later turn: $invalid", + async ({ invalid }) => { + const trace = await queryTrace([ + span("root", { "gen_ai.response.tool_calls": JSON.stringify(calls) }), + span("later", { "gen_ai.response.tool_calls": invalid }, "root"), + ]); + expect(trace?.response?.toolCalls).toEqual(calls); + }, +); + +test.each( + [calls, JSON.stringify(calls), []].map((toolCalls) => ({ toolCalls })), +)( + "GenAI queries accept array response tool calls: $toolCalls", + async ({ toolCalls }) => { + const trace = await queryTrace([ + span("root", { "gen_ai.response.tool_calls": toolCalls }), + ]); + expect(trace?.response?.toolCalls).toEqual( + typeof toolCalls === "string" ? calls : toolCalls, + ); + }, +); + +test.each(["gen_ai.prompt", "vercel.ai.prompt"])( + "Vercel AI queries prefer structured request messages over %s", + async (field) => { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + [field]: JSON.stringify({ prompt: "fallback", system: "Be concise" }), + "gen_ai.request.messages": JSON.stringify([ + { role: "user", content: [{ type: "text", text: "Structured" }] }, + ]), + }), + ]); + expect(trace?.prompt).toEqual({ + messages: [{ role: "user", content: "Structured" }], + system: "Be concise", + }); + }, +); + +test.each(["gen_ai.prompt", "vercel.ai.prompt"])( + "Vercel AI queries normalize the text prompt in %s", + async (field) => { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + [field]: JSON.stringify({ + prompt: "Fallback text", + system: "Be concise", + }), + }), + ]); + expect(trace?.prompt).toEqual({ + messages: [{ role: "user", content: "Fallback text" }], + system: "Be concise", + }); + }, +); + +test("Vercel AI queries prefer a child's structured messages over the root fallback", async () => { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.prompt": JSON.stringify({ + prompt: "fallback", + system: "Be concise", + }), + }), + span( + "first-turn", + { + "gen_ai.request.messages": JSON.stringify([ + { role: "user", content: "Initial conversation" }, + ]), + }, + "root", + ), + ]); + expect(trace?.prompt).toEqual({ + messages: [{ role: "user", content: "Initial conversation" }], + system: "Be concise", + }); +}); + +test("Vercel AI queries keep the first explicit request messages across child turns", async () => { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.prompt": JSON.stringify({ + prompt: "fallback", + system: "Be concise", + }), + }), + span( + "first-turn", + { + "gen_ai.request.messages": JSON.stringify([ + { role: "user", content: "Initial conversation" }, + ]), + }, + "root", + ), + span( + "later-turn", + { + "gen_ai.request.messages": JSON.stringify([ + { role: "user", content: "Later conversation" }, + ]), + }, + "root", + ), + ]); + expect(trace?.prompt).toEqual({ + messages: [{ role: "user", content: "Initial conversation" }], + system: "Be concise", + }); +}); + +test.each([ + { + prompt: "Raw fallback", + expected: { messages: [{ role: "unknown", content: "Raw fallback" }] }, + }, + { + prompt: JSON.stringify({ + messages: [ + { role: "user", content: [{ type: "text", text: "Fallback" }] }, + ], + system: "Be concise", + }), + expected: { + messages: [{ role: "user", content: "Fallback" }], + system: "Be concise", + }, + }, + { + prompt: JSON.stringify({ system: "Be concise" }), + expected: { system: "Be concise" }, + }, + { + prompt: JSON.stringify({ + messages: null, + prompt: "Fallback text", + system: "Be concise", + }), + expected: { + messages: [{ role: "user", content: "Fallback text" }], + system: "Be concise", + }, + }, +])( + "Vercel AI queries preserve fallback prompts: $prompt", + async ({ prompt, expected }) => { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.prompt": prompt, + }), + ]); + expect(trace?.prompt).toEqual(expected); + }, +); + +test("Vercel AI queries retain token usage on leaf spans without children", async () => { + const trace = await queryTrace([ + span("leaf", { + "vercel.ai.operationId": "ai.generateText", + "gen_ai.usage.input_tokens": 12, + "gen_ai.usage.output_tokens": 3, + }), + ]); + expect(trace).toMatchObject({ + promptTokens: 12, + completionTokens: 3, + tokensDisplay: "12 / 3", + }); +}); diff --git a/packages/spotlight/src/telemetry/ai/genAI.ts b/packages/spotlight/src/telemetry/ai/genAI.ts index 1d96dabba..5db0fdfa9 100644 --- a/packages/spotlight/src/telemetry/ai/genAI.ts +++ b/packages/spotlight/src/telemetry/ai/genAI.ts @@ -354,10 +354,11 @@ function extractResponseData( const toolCalls = span.data[GEN_AI_RESPONSE_TOOL_CALLS_FIELD]; if (toolCalls) { try { - trace.response.toolCalls = - typeof toolCalls === "string" - ? JSON.parse(toolCalls) - : (toolCalls as AIToolCall[]); + const parsed: unknown = + typeof toolCalls === "string" ? JSON.parse(toolCalls) : toolCalls; + if (Array.isArray(parsed)) { + trace.response.toolCalls = parsed; + } } catch { // ignore unparseable tool call payloads } diff --git a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts index fa32d4dff..0d1e69eea 100644 --- a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts +++ b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts @@ -1,8 +1,15 @@ import { stringifyValue } from "../utils/stringifyValue.ts"; import { finiteNumber } from "../utils/finiteNumber.ts"; import { getAllSpansInTree } from "../store/helpers.ts"; -import type { AILibraryHandler, AIToolCall, AITrace, Span } from "../types.ts"; +import type { + AILibraryHandler, + AIPrompt, + AIToolCall, + AITrace, + Span, +} from "../types.ts"; import { extractFinishReason } from "./finishReason.ts"; +import { parsed, record } from "../internal/wireValues.ts"; // https://ai-sdk.dev/docs/ai-sdk-core/telemetry const AI_SPAN_OP_PREFIX = "gen_ai."; @@ -316,7 +323,7 @@ function normalizeMessageContent(content: unknown): string { return content .filter( (block): block is { type: string; text: string } => - block.type === "text" && typeof block.text === "string", + block?.type === "text" && typeof block.text === "string", ) .map((block) => block.text) .join(""); @@ -324,14 +331,31 @@ function normalizeMessageContent(content: unknown): string { return ""; } +function normalizePromptMessages(value: unknown): AIPrompt["messages"] { + const messages = parsed(value); + if (!Array.isArray(messages)) return undefined; + const normalized: NonNullable = []; + for (const value of messages) { + const message = record(value); + if (typeof message.role !== "string") return undefined; + normalized.push({ + ...message, + role: message.role, + content: normalizeMessageContent(message.content), + }); + } + return normalized; +} + function parseSpanData(spans: Span[], trace: AITrace) { + const promptState = { hasRequestMessages: false }; const responseState = { finishReasonFromText: false }; for (const span of spans) { if (!span.data) continue; extractAIMetadata(span, trace); extractTelemetryMetadata(span, trace); - extractPromptData(span, trace); + extractPromptData(span, trace, promptState); extractResponseData(span, trace, responseState); extractToolCallData(span, trace); } @@ -386,49 +410,43 @@ function extractTelemetryMetadata(span: Span, trace: AITrace) { } } -function extractPromptData(span: Span, trace: AITrace) { - if (!span.data) return; +function extractPromptData( + span: Span, + trace: AITrace, + state: { hasRequestMessages: boolean }, +) { + if (!span.data || state.hasRequestMessages) return; + const requestMessages = span.data[AI_PROMPT_MESSAGES_FIELD]; const promptField = extractFieldFromSpan(span.data, "PROMPT"); - if (promptField && !trace.prompt) { - try { - const parsed = JSON.parse(stringifyValue(promptField)); - // Normalize message content from v2 array format to string - if (parsed.messages && Array.isArray(parsed.messages)) { - parsed.messages = parsed.messages.map( - (msg: { role: string; content: unknown }) => ({ - ...msg, - content: normalizeMessageContent(msg.content), - }), - ); - } - trace.prompt = parsed; - } catch { - trace.prompt = { - messages: [{ role: "unknown", content: stringifyValue(promptField) }], - }; - } + const promptValue = parsed(promptField); + const prompt = record(promptValue); + const system = + typeof prompt.system === "string" ? prompt.system : trace.prompt?.system; + const structured = normalizePromptMessages(requestMessages); + if (structured !== undefined) { + trace.prompt = { messages: structured, system }; + state.hasRequestMessages = true; + return; } - - const promptMessages = span.data[AI_PROMPT_MESSAGES_FIELD]; - if (promptMessages && !trace.prompt) { - try { - const messages = JSON.parse(stringifyValue(promptMessages)); - // Normalize message content from v2 array format to string - const normalizedMessages = messages.map( - (msg: { role: string; content: unknown }) => ({ - ...msg, - content: normalizeMessageContent(msg.content), - }), - ); - trace.prompt = { messages: normalizedMessages }; - } catch { + if (trace.prompt) return; + + const fallback = normalizePromptMessages(prompt.messages); + if (fallback !== undefined) { + trace.prompt = { messages: fallback, system }; + } else if (typeof prompt.prompt === "string") { + trace.prompt = { + messages: [{ role: "user", content: prompt.prompt }], + system, + }; + } else if (system !== undefined) { + trace.prompt = { system }; + } else { + const raw = requestMessages ?? promptValue; + if (raw !== undefined && raw !== null) trace.prompt = { - messages: [ - { role: "unknown", content: stringifyValue(promptMessages) }, - ], + messages: [{ role: "unknown", content: stringifyValue(raw) }], }; - } } } diff --git a/packages/spotlight/src/telemetry/data/profiles.test.ts b/packages/spotlight/src/telemetry/data/profiles.test.ts index 13acb45fe..d4c833edc 100644 --- a/packages/spotlight/src/telemetry/data/profiles.test.ts +++ b/packages/spotlight/src/telemetry/data/profiles.test.ts @@ -252,13 +252,14 @@ describe("profile graft thread selection", () => { expect(target.spans.get(spans[3].span_id)).toBe(spans[3]); }); - it("keeps gap bounds when a same-thread next sample falls beyond the gap", () => { + it("clips an in-gap sample when its same-thread next sample falls beyond the gap", () => { const spans = getSpansFromProfile( trace(), profile({ samples: [ { thread_id: "main", stack_id: 0, start_timestamp: 0 }, { thread_id: "main", stack_id: 0, start_timestamp: 4 }, + { thread_id: "worker", stack_id: 0, start_timestamp: 6 }, { thread_id: "main", stack_id: 0, start_timestamp: 12 }, ], }), @@ -267,7 +268,9 @@ describe("profile graft thread selection", () => { 8, new Set(), ); - expect(spans).toEqual([]); + expect(spans).toMatchObject([ + { op: "main", start_timestamp: 4, timestamp: 8 }, + ]); }); it.each(["previous", "next", "parent"])( diff --git a/packages/spotlight/src/telemetry/data/profiles.ts b/packages/spotlight/src/telemetry/data/profiles.ts index 0915a2ebb..b4d4f6ddb 100644 --- a/packages/spotlight/src/telemetry/data/profiles.ts +++ b/packages/spotlight/src/telemetry/data/profiles.ts @@ -162,11 +162,7 @@ export function getSpansFromProfile( if (sampleTs < startTs || sampleTs > endTs) { continue; } - const timestamp = nextSampleTimestamps[sampleIdx] ?? endTs; - - if (timestamp > endTs) { - continue; - } + const timestamp = Math.min(nextSampleTimestamps[sampleIdx] ?? endTs, endTs); const commonAttributes = { start_timestamp: sampleTs, timestamp, diff --git a/packages/spotlight/src/telemetry/profile-intervals.spec.ts b/packages/spotlight/src/telemetry/profile-intervals.spec.ts new file mode 100644 index 000000000..afaf2f6b3 --- /dev/null +++ b/packages/spotlight/src/telemetry/profile-intervals.spec.ts @@ -0,0 +1,91 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "../sdk/core.ts"; + +test("public ingestion clips a profile interval at the next real span", async () => { + const store = createTelemetryStore(); + try { + expect( + ( + await store.ingestEnvelope([ + {}, + [ + [ + { type: "transaction" }, + { + event_id: "transaction", + type: "transaction", + transaction: "profile interval", + start_timestamp: 1, + timestamp: 15, + contexts: { + trace: { trace_id: "trace", span_id: "root" }, + }, + spans: [ + { + span_id: "left", + parent_span_id: "root", + start_timestamp: 1, + timestamp: 3, + }, + { + span_id: "right", + parent_span_id: "root", + start_timestamp: 9, + timestamp: 15, + }, + ], + }, + ], + [ + { type: "profile" }, + { + event_id: "profile", + version: "1", + timestamp: 1, + platform: "javascript", + transaction: { + trace_id: "trace", + active_thread_id: "main", + }, + profile: { + frames: [{ module: "app", function: "work" }], + stacks: [[0]], + samples: [ + { + thread_id: "main", + stack_id: 0, + elapsed_since_start_ns: "4000000000", + }, + { + thread_id: "main", + stack_id: 0, + elapsed_since_start_ns: "12000000000", + }, + ], + }, + }, + ], + ], + ]) + ).status, + ).toBe("processed"); + const trace = store.getTrace("trace")!; + expect(trace.spans.get("root")?.children).toMatchObject([ + { span_id: "left", start_timestamp: 1000, timestamp: 3000 }, + { + op: "work", + start_timestamp: 5000, + timestamp: 9000, + tags: { source: "profile" }, + }, + { + span_id: "right", + start_timestamp: 9000, + timestamp: 15000, + children: [{ op: "work", start_timestamp: 13000, timestamp: 15000 }], + }, + ]); + } finally { + store.dispose(); + } +}); From 857a36fd07853d819151e457252d298802237319 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 12:52:20 +0530 Subject: [PATCH 06/15] ref(spotlight): Remove unused processing internals Remove legacy subscriptions, disabled logging, unused private helpers and types, and unnecessary async test callbacks. Preserve the public API and executable examples. Co-Authored-By: Codex --- .../src/components/event-detail.test.tsx | 16 ++++---- .../investigation-controls.test.tsx | 2 +- .../components/performance-workspace.test.tsx | 4 +- apps/local/src/lib/performance.test.ts | 6 +-- apps/local/src/lib/trace-model.test.ts | 2 +- .../spotlight/src/telemetry/ai/aiLibraries.ts | 12 +----- .../spotlight/src/telemetry/data/profiles.ts | 2 - .../src/telemetry/envelope/userAgent.ts | 26 ------------ .../src/telemetry/internal/engine.ts | 2 - packages/spotlight/src/telemetry/logger.ts | 23 ----------- packages/spotlight/src/telemetry/queries.ts | 2 - .../telemetry/store/slices/envelopesSlice.ts | 9 ----- .../src/telemetry/store/slices/eventsSlice.ts | 8 ---- .../src/telemetry/store/slices/logsSlice.ts | 6 --- .../telemetry/store/slices/profilesSlice.ts | 23 ++--------- .../src/telemetry/store/slices/sharedSlice.ts | 14 +------ .../store/slices/subscriptionsSlice.ts | 32 --------------- .../spotlight/src/telemetry/store/types.ts | 40 +------------------ .../telemetry/store/utils/traceProcessor.ts | 6 --- packages/spotlight/src/telemetry/types.ts | 22 ---------- .../spotlight/src/telemetry/utils/sentry.ts | 12 ------ .../spotlight/src/telemetry/utils/traces.ts | 39 +----------------- 22 files changed, 22 insertions(+), 286 deletions(-) delete mode 100644 packages/spotlight/src/telemetry/logger.ts delete mode 100644 packages/spotlight/src/telemetry/queries.ts delete mode 100644 packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts diff --git a/apps/local/src/components/event-detail.test.tsx b/apps/local/src/components/event-detail.test.tsx index ce8ee8a6b..bcaaecd6e 100644 --- a/apps/local/src/components/event-detail.test.tsx +++ b/apps/local/src/components/event-detail.test.tsx @@ -17,7 +17,7 @@ function item(type: string, payload: unknown) { return entry; } -test("logs display their message, severity, and structured attributes", async () => { +test("logs display their message, severity, and structured attributes", () => { render( { +test("raw-only SDK payloads default to JSON and reset on a different item", () => { const { rerender } = render( , ); @@ -61,7 +61,7 @@ test("raw-only SDK payloads default to JSON and reset on a different item", asyn ).toBe("true"); }); -test("arrow navigation wraps between detail tabs and moves keyboard focus", async () => { +test("arrow navigation wraps between detail tabs and moves keyboard focus", () => { render(); const overview = screen.getByRole("tab", { name: "Overview" }); fireEvent.keyDown(overview, { key: "ArrowLeft" }); @@ -73,7 +73,7 @@ test("arrow navigation wraps between detail tabs and moves keyboard focus", asyn expect(document.activeElement).toBe(overview); }); -test("raw envelopes retain their recognizable heading and open directly in JSON", async () => { +test("raw envelopes retain their recognizable heading and open directly in JSON", () => { const envelope = { id: "envelope:1", type: "envelope", @@ -91,7 +91,7 @@ test("raw envelopes retain their recognizable heading and open directly in JSON" ); }); -test("malformed and nested log attributes remain readable inert text", async () => { +test("malformed and nested log attributes remain readable inert text", () => { render( { +test("requests show a compact copyable summary with captured method and URL", () => { const copied: string[] = []; vi.stubGlobal("navigator", { clipboard: { @@ -155,7 +155,7 @@ test("requests show a compact copyable summary with captured method and URL", as expect(copied).toEqual(["201"]); }); -test("span summaries use captured typed URL attributes without inventing request fields", async () => { +test("span summaries use captured typed URL attributes without inventing request fields", () => { render( { +test("captured context suppresses the empty-details message", () => { render( { +test("log filters combine severity and logger and sort timestamps without changing captured data", () => { expect( filterEvents(events, { ...defaultEventFilters, diff --git a/apps/local/src/components/performance-workspace.test.tsx b/apps/local/src/components/performance-workspace.test.tsx index 6e6a7e0ac..268f796bd 100644 --- a/apps/local/src/components/performance-workspace.test.tsx +++ b/apps/local/src/components/performance-workspace.test.tsx @@ -47,7 +47,7 @@ test("resource groups sort and open the selected execution in its trace", async fireEvent.click(screen.getByRole("button", { name: /Page.*two.*100.0 ms/ })); expect(onSelect).toHaveBeenCalledWith("t", "two"); }); -test("Web Vitals metric switching displays captured INP without fabricating missing visits", async () => { +test("Web Vitals metric switching displays captured INP without fabricating missing visits", () => { render( { +test("profile chart supports zoom, visualization switching and independent thread samples", () => { const profile = item("profile", { profile: { frames: [{ function: "mainWork" }, { function: "workerWork" }], diff --git a/apps/local/src/lib/performance.test.ts b/apps/local/src/lib/performance.test.ts index 69a410e6f..08908c9b3 100644 --- a/apps/local/src/lib/performance.test.ts +++ b/apps/local/src/lib/performance.test.ts @@ -39,7 +39,7 @@ test("query analysis groups embedded and standalone executions without counting span: { id: "q1" }, }); }); -test("Web Vitals retain per-visit units and classify thresholds instead of inventing missing measurements", async () => { +test("Web Vitals retain per-visit units and classify thresholds instead of inventing missing measurements", () => { const [page] = webVitalPages([ decode("transaction", { transaction: "/orders", @@ -56,7 +56,7 @@ test("Web Vitals retain per-visit units and classify thresholds instead of inven }); expect(page?.visits[0]?.metrics.inp).toBeUndefined(); }); -test("profiles build root-first trees, inclusive/self samples, and interval times per thread", async () => { +test("profiles build root-first trees, inclusive/self samples, and interval times per thread", () => { const payload = { profile: { frames: [{ function: "main" }, { function: "query" }], @@ -81,7 +81,7 @@ test("profiles build root-first trees, inclusive/self samples, and interval time }); expect(profile.threads).toHaveLength(2); }); -test("continuous chunks and missing samples remain honest", async () => { +test("continuous chunks and missing samples remain honest", () => { const chunk = parseProfile( decode("profile_chunk", { profile: { diff --git a/apps/local/src/lib/trace-model.test.ts b/apps/local/src/lib/trace-model.test.ts index 185afd32b..f2cfbcdf3 100644 --- a/apps/local/src/lib/trace-model.test.ts +++ b/apps/local/src/lib/trace-model.test.ts @@ -215,7 +215,7 @@ describe("buildTraceGroups", () => { expect(trace?.roots[0]?.children).toHaveLength(70000); }); - test("uses a transaction ID as the trace fallback used by routing", async () => { + test("uses a transaction ID as the trace fallback used by routing", () => { expect( traceIdForItem( item({ diff --git a/packages/spotlight/src/telemetry/ai/aiLibraries.ts b/packages/spotlight/src/telemetry/ai/aiLibraries.ts index 219f9306d..889a5aa99 100644 --- a/packages/spotlight/src/telemetry/ai/aiLibraries.ts +++ b/packages/spotlight/src/telemetry/ai/aiLibraries.ts @@ -1,4 +1,4 @@ -import type { AILibraryHandler, AITrace, Span, Trace } from "../types.ts"; +import type { AILibraryHandler, AITrace, Span } from "../types.ts"; import { genAIHandler } from "./genAI.ts"; import { vercelAISDKHandler } from "./vercelAISDK.ts"; @@ -50,13 +50,3 @@ export function createAITraceFromSpan(span: Span): AITrace | null { return handler.processTrace(span); } - -export function hasAISpans(trace: Trace): boolean { - if (!trace.spanTree) { - return false; - } - - // TODO: We may want to cache/optimize this `extractAllAIRootSpans` helper - const aiRootSpans = extractAllAIRootSpans(trace.spanTree); - return aiRootSpans.length > 0; -} diff --git a/packages/spotlight/src/telemetry/data/profiles.ts b/packages/spotlight/src/telemetry/data/profiles.ts index b4d4f6ddb..7575046df 100644 --- a/packages/spotlight/src/telemetry/data/profiles.ts +++ b/packages/spotlight/src/telemetry/data/profiles.ts @@ -1,4 +1,3 @@ -import { log } from "../logger.ts"; import type { SentryProfileWithTraceMeta } from "../store/types.ts"; import type { Span, Trace } from "../types.ts"; import { getFunctionNameFromFrame } from "../utils/profileUtils.ts"; @@ -299,7 +298,6 @@ function graftProfileSpansIntoTree( spanTree: Span[] = trace.spanTree, parent: Span | Trace = trace, ) { - log(`Grafting profile spans into trace ${trace.trace_id}`); const levels: { spanTree: Span[]; parent: Span | Trace; index: number }[] = [ { spanTree, parent, index: -1 }, ]; diff --git a/packages/spotlight/src/telemetry/envelope/userAgent.ts b/packages/spotlight/src/telemetry/envelope/userAgent.ts index 5762c5f94..26382f160 100644 --- a/packages/spotlight/src/telemetry/envelope/userAgent.ts +++ b/packages/spotlight/src/telemetry/envelope/userAgent.ts @@ -33,29 +33,3 @@ export function parseBrowserFromUserAgent(userAgent: string): string { const match = userAgent.match(/^([^/\s(]+)/); return match ? match[1] : "unknown"; } - -/** - * Builds a client identifier from browser info and optional hostname - * Format: clientName/version (metadata) - * Example: "ui/Chrome/131 (example.com)" - */ -export function buildClientIdentifier( - clientName: string, - userAgent?: string, - hostname?: string, -): string { - let clientId = clientName; - - if (userAgent) { - const browserInfo = parseBrowserFromUserAgent(userAgent); - if (browserInfo !== "unknown") { - clientId = `${clientName}/${browserInfo}`; - } - } - - if (hostname && hostname !== "localhost") { - clientId += ` (${hostname})`; - } - - return clientId; -} diff --git a/packages/spotlight/src/telemetry/internal/engine.ts b/packages/spotlight/src/telemetry/internal/engine.ts index cb0600815..ba9ddba22 100644 --- a/packages/spotlight/src/telemetry/internal/engine.ts +++ b/packages/spotlight/src/telemetry/internal/engine.ts @@ -5,7 +5,6 @@ import { createLogsSlice } from "../store/slices/logsSlice.ts"; import { createProfilesSlice } from "../store/slices/profilesSlice.ts"; import { createSDKsSlice } from "../store/slices/sdksSlice.ts"; import { createSharedSlice } from "../store/slices/sharedSlice.ts"; -import { createSubscriptionsSlice } from "../store/slices/subscriptionsSlice.ts"; import { createTracesSlice } from "../store/slices/tracesSlice.ts"; import type { SentryStore } from "../store/types.ts"; @@ -20,7 +19,6 @@ export function createEngine(options: EngineOptions = {}) { ...createEventsSlice(options, runtime)(...args), ...createTracesSlice(...args), ...createProfilesSlice(...args), - ...createSubscriptionsSlice(...args), ...createEnvelopesSlice(runtime)(...args), ...createSDKsSlice(...args), ...createLogsSlice(...args), diff --git a/packages/spotlight/src/telemetry/logger.ts b/packages/spotlight/src/telemetry/logger.ts deleted file mode 100644 index c9451093d..000000000 --- a/packages/spotlight/src/telemetry/logger.ts +++ /dev/null @@ -1,23 +0,0 @@ -export const TELEMETRY_PREFIX = "🔎 [Telemetry]"; - -const noop = (..._args: unknown[]) => {}; -let _log = noop; -let _warn = noop; - -export function activateLogger() { - _log = (...args: unknown[]) => console.log(TELEMETRY_PREFIX, ...args); - _warn = (...args: unknown[]) => console.warn(TELEMETRY_PREFIX, ...args); -} - -export function deactivateLogger() { - _log = noop; - _warn = noop; -} - -export function log(...args: unknown[]) { - _log(...args); -} - -export function warn(...args: unknown[]) { - _warn(...args); -} diff --git a/packages/spotlight/src/telemetry/queries.ts b/packages/spotlight/src/telemetry/queries.ts deleted file mode 100644 index 5684ae41c..000000000 --- a/packages/spotlight/src/telemetry/queries.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Internal compatibility barrel. Prefer importing from the individual selector domain. -export * from "./selectors/index.ts"; diff --git a/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts b/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts index 8f4b09390..16bac63b0 100644 --- a/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts +++ b/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts @@ -187,12 +187,6 @@ export const createEnvelopesSlice = } return { envelopes: new Map(), - // Compatibility path used by the UI: synchronous processing except optional enrichment. - pushEnvelope: (envelope) => { - const { count, pending } = process(envelope); - for (const task of pending) void task.catch(() => {}); - return count; - }, ingestEnvelope: async (envelope) => { const { count, pending } = process(envelope); const results = await Promise.allSettled(pending); @@ -200,9 +194,6 @@ export const createEnvelopesSlice = if (failure?.status === "rejected") throw failure.reason; return count; }, - getEnvelopeById: (id: string) => { - return get().envelopes.get(id); - }, getEnvelopes: () => Array.from(get().envelopes.values()), }; }; diff --git a/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts index adf67ede2..253299e92 100644 --- a/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts +++ b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts @@ -180,14 +180,6 @@ export const createEventsSlice = set({ eventsById: newEventIds }); if (runtime.generation !== generation) return; - // Notify event subscribers - for (const [type, callback] of get().subscribers.values()) { - if (type === "event") { - (callback as (event: SentryEvent) => void)(event); - if (runtime.generation !== generation) return; - } - } - const traceCtx = event.contexts?.trace; if (traceCtx?.trace_id) { const { tracesById, profilesByTraceId, profileChunksByProfilerId } = diff --git a/packages/spotlight/src/telemetry/store/slices/logsSlice.ts b/packages/spotlight/src/telemetry/store/slices/logsSlice.ts index 32c960b6a..35d2490ca 100644 --- a/packages/spotlight/src/telemetry/store/slices/logsSlice.ts +++ b/packages/spotlight/src/telemetry/store/slices/logsSlice.ts @@ -13,11 +13,5 @@ export const createLogsSlice: StateCreator< > = (_set, get) => ({ logsById: new Map(), logsByTraceId: new Map(), - - getLogById: (id: string) => get().logsById.get(id), getLogs: () => Array.from(get().logsById.values()), - getLogsByTraceId: (traceId: string) => { - const logsByTraceId = get().logsByTraceId.get(traceId); - return logsByTraceId ? Array.from(logsByTraceId) : []; - }, }); diff --git a/packages/spotlight/src/telemetry/store/slices/profilesSlice.ts b/packages/spotlight/src/telemetry/store/slices/profilesSlice.ts index 9536388ac..9fe776389 100644 --- a/packages/spotlight/src/telemetry/store/slices/profilesSlice.ts +++ b/packages/spotlight/src/telemetry/store/slices/profilesSlice.ts @@ -1,29 +1,12 @@ import type { StateCreator } from "zustand/vanilla"; -import { getProfileFunctionSummaries } from "../../selectors/profiles.ts"; -import type { - ProfilesSliceActions, - ProfilesSliceState, - SentryStore, -} from "../types.ts"; +import type { ProfilesSliceState, SentryStore } from "../types.ts"; export const createProfilesSlice: StateCreator< SentryStore, [], [], - ProfilesSliceState & ProfilesSliceActions -> = (_set, get) => ({ + ProfilesSliceState +> = () => ({ profilesByTraceId: new Map(), profileChunksByProfilerId: new Map(), - - // Unified profile access - works for both V1 and merged V2 profiles - getProfileByTraceId: (id: string) => get().profilesByTraceId.get(id), - - // Keep legacy field names private to the store adapter. - getAggregateCallData: () => - getProfileFunctionSummaries(get().profilesByTraceId).map((summary) => ({ - name: summary.name, - totalTime: summary.totalDurationMs, - samples: summary.sampleCount, - traceIds: summary.traceIds, - })), }); diff --git a/packages/spotlight/src/telemetry/store/slices/sharedSlice.ts b/packages/spotlight/src/telemetry/store/slices/sharedSlice.ts index 9c5d2a9a5..5789594ae 100644 --- a/packages/spotlight/src/telemetry/store/slices/sharedSlice.ts +++ b/packages/spotlight/src/telemetry/store/slices/sharedSlice.ts @@ -6,19 +6,7 @@ export const createSharedSlice = ( runtime: ProcessingRuntime, ): StateCreator => - (set, get) => ({ - getEventById: (id: string) => get().eventsById.get(id), - getTraceById: (id: string) => get().tracesById.get(id), - getEventsByTrace: (traceId: string, spanId?: string | null) => - get() - .getEvents() - .filter((event) => { - const trace = event.contexts?.trace; - return ( - trace?.trace_id === traceId && - (spanId === undefined || trace.span_id === spanId) - ); - }), + (set) => ({ resetData: () => { runtime.generation += 1; set({ diff --git a/packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts b/packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts deleted file mode 100644 index 9129cf4e7..000000000 --- a/packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { StateCreator } from "zustand/vanilla"; -import { generateUuidv4 } from "../../uuid.ts"; -import type { - SentryStore, - Subscription, - SubscriptionsSliceActions, - SubscriptionsSliceState, -} from "../types.ts"; - -export const createSubscriptionsSlice: StateCreator< - SentryStore, - [], - [], - SubscriptionsSliceState & SubscriptionsSliceActions -> = (set, get) => ({ - subscribers: new Map(), - - subscribe: (...args: Subscription) => { - const id = generateUuidv4(); - const { subscribers } = get(); - const newSubscribers = new Map(subscribers); - newSubscribers.set(id, args); - set({ subscribers: newSubscribers }); - - return () => { - const { subscribers: currentSubscribers } = get(); - const updatedSubscribers = new Map(currentSubscribers); - updatedSubscribers.delete(id); - set({ subscribers: updatedSubscribers }); - }; - }, -}); diff --git a/packages/spotlight/src/telemetry/store/types.ts b/packages/spotlight/src/telemetry/store/types.ts index 38ecc9815..fbe925cba 100644 --- a/packages/spotlight/src/telemetry/store/types.ts +++ b/packages/spotlight/src/telemetry/store/types.ts @@ -1,24 +1,10 @@ import type { Envelope } from "@sentry/core"; -import type { - AggregateCallData, - Sdk, - SentryEvent, - SentryLogEventItem, - Trace, -} from "../types.ts"; +import type { Sdk, SentryEvent, SentryLogEventItem, Trace } from "../types.ts"; import type { ProcessedProfileChunk } from "./utils/profileChunkProcessor.ts"; import type { SentryProfileWithTraceMeta } from "../types.ts"; export type { SentryProfileWithTraceMeta } from "../types.ts"; -export type OnlineSubscription = ["online", (status: boolean) => void]; -export type EventSubscription = ["event", (event: SentryEvent) => void]; -export type TraceSubscription = ["trace", (trace: Trace) => void]; -export type Subscription = - | OnlineSubscription - | EventSubscription - | TraceSubscription; - export interface EventsSliceState { eventsById: Map; } @@ -43,28 +29,12 @@ export interface ProfilesSliceState { profileChunksByProfilerId: Map; } -export interface ProfilesSliceActions { - /** Get profile for a trace - unified access for both V1 and V2 profiles */ - getProfileByTraceId: (id: string) => SentryProfileWithTraceMeta | undefined; - getAggregateCallData: () => AggregateCallData[]; -} - -export interface SubscriptionsSliceState { - subscribers: Map; -} - -export interface SubscriptionsSliceActions { - subscribe: (...args: Subscription) => () => void; -} - export interface EnvelopesSliceState { envelopes: Map; } export interface EnvelopesSliceActions { - pushEnvelope: (params: Envelope) => number; ingestEnvelope: (params: Envelope) => Promise; - getEnvelopeById: (id: string) => Envelope | undefined; getEnvelopes: () => Array; } @@ -78,9 +48,7 @@ export interface LogsSliceState { } export interface LogsSliceActions { - getLogById: (id: string) => SentryLogEventItem | undefined; getLogs: () => SentryLogEventItem[]; - getLogsByTraceId: (traceId: string) => SentryLogEventItem[]; } export interface SDKsSliceActions { @@ -90,24 +58,18 @@ export interface SDKsSliceActions { } export interface SharedSliceActions { - getEventById: (id: string) => SentryEvent | undefined; - getTraceById: (id: string) => Trace | undefined; - getEventsByTrace: (traceId: string, spanId?: string | null) => SentryEvent[]; resetData: () => void; } export type SentryStoreState = EventsSliceState & TracesSliceState & ProfilesSliceState & - SubscriptionsSliceState & EnvelopesSliceState & LogsSliceState & SDKsSliceState; export type SentryStoreActions = EventsSliceActions & TracesSliceActions & - ProfilesSliceActions & - SubscriptionsSliceActions & EnvelopesSliceActions & LogsSliceActions & SDKsSliceActions & diff --git a/packages/spotlight/src/telemetry/store/utils/traceProcessor.ts b/packages/spotlight/src/telemetry/store/utils/traceProcessor.ts index 7b8c0c3c7..9d2e2e3c7 100644 --- a/packages/spotlight/src/telemetry/store/utils/traceProcessor.ts +++ b/packages/spotlight/src/telemetry/store/utils/traceProcessor.ts @@ -1,5 +1,4 @@ import { graftProfileSpans } from "../../data/profiles.ts"; -import { log } from "../../logger.ts"; import type { SentryTransactionEvent, Span, Trace } from "../../types.ts"; import { compareSpans, groupSpans } from "../../utils/traces.ts"; import type { SentryProfileWithTraceMeta } from "../types.ts"; @@ -201,11 +200,6 @@ export function updateTraceMetadata(trace: Trace): void { } else if (roots.length > 1) { trace.rootTransactionName = "(multiple root transactions)"; } else if (trace.transactions.length > 0) { - // Orphan trace: no root transaction, but has child transactions - log( - `[Telemetry] Orphan trace detected (trace_id: ${trace.trace_id}). ` + - `Using first transaction "${trace.transactions[0].transaction}" as fallback.`, - ); // use the first transcation for orphan traces trace.rootTransaction = trace.transactions[0]; trace.rootTransactionName = diff --git a/packages/spotlight/src/telemetry/types.ts b/packages/spotlight/src/telemetry/types.ts index 061d6ee63..aa758340c 100644 --- a/packages/spotlight/src/telemetry/types.ts +++ b/packages/spotlight/src/telemetry/types.ts @@ -201,13 +201,6 @@ export type SentryProfileWithTraceMeta = SentryProcessedProfile & { active_thread_id: string; }; -export type AggregateCallData = { - name: string; - totalTime: number; - samples: number; - traceIds: Set; -}; - export type SentryDeviceInfo = { architecture: string; is_emulator?: boolean; @@ -405,21 +398,6 @@ export type SentryEventWithPerformanceData = Omit< }; }; -export type MetricScoreProps = { - fcpScore: number; - lcpScore: number; - clsScore: number; - fidScore: number; - ttfbScore: number; -}; -export type MetricWeightsProps = { - fcp: number; - lcp: number; - cls: number; - fid: number; - ttfb: number; -}; - export type AIToolCall = { toolCallId: string; toolName: string; diff --git a/packages/spotlight/src/telemetry/utils/sentry.ts b/packages/spotlight/src/telemetry/utils/sentry.ts index e6b046d76..72054eb05 100644 --- a/packages/spotlight/src/telemetry/utils/sentry.ts +++ b/packages/spotlight/src/telemetry/utils/sentry.ts @@ -1,6 +1,5 @@ import { ERROR_EVENT_TYPES, - FEEDBACK_EVENT_TYPES, LOG_EVENT_TYPES, PROFILE_EVENT_TYPES, TRACE_EVENT_TYPES, @@ -8,7 +7,6 @@ import { import type { SentryErrorEvent, SentryEvent, - SentryFeedbackEvent, SentryLogEvent, SentryProfileV1Event, SentryProfileV2ChunkEvent, @@ -50,13 +48,3 @@ export function isTraceEvent( export function isLogEvent(event: SentryEvent): event is SentryLogEvent { return !!event.type && LOG_EVENT_TYPES.has(event.type); } - -export function isFeedbackEvent( - event: SentryEvent, -): event is SentryFeedbackEvent { - return ( - !!event.type && - FEEDBACK_EVENT_TYPES.has(event.type) && - Boolean((event as SentryFeedbackEvent).contexts?.feedback) - ); -} diff --git a/packages/spotlight/src/telemetry/utils/traces.ts b/packages/spotlight/src/telemetry/utils/traces.ts index 874305ef4..81b4ce1eb 100644 --- a/packages/spotlight/src/telemetry/utils/traces.ts +++ b/packages/spotlight/src/telemetry/utils/traces.ts @@ -1,6 +1,4 @@ -import { stringifyValue } from "./stringifyValue.ts"; -import { log } from "../logger.ts"; -import type { Span, Trace } from "../types.ts"; +import type { Span } from "../types.ts"; // mutates spans in place and adds children, as well as returns the top level tree export function groupSpans(spans: Map): Span[] { @@ -21,15 +19,6 @@ export function groupSpans(spans: Map): Span[] { parent.children.push(span); } else if (span.parent_span_id && span.parent_span_id !== span.span_id) { const parentParent = sortedSpans.find((s) => !s.parent_span_id); - if (!parentParent) { - log( - `Root span (${span.parent_span_id}) for span (${span.span_id}). Creating orphan.`, - ); - } else { - log( - `Creating orphan for parent (${span.parent_span_id}) for span (${span.span_id})`, - ); - } parent = { trace_id: span.trace_id, span_id: span.parent_span_id, @@ -79,29 +68,3 @@ export function compareSpans( ): number { return a.start_timestamp - b.start_timestamp; } - -export function getRootTransactionMethod(trace: Trace) { - // Fallback to first transaction if no root transaction (e.g., orphan traces) - const transaction = trace.rootTransaction || trace.transactions[0]; - const method = stringifyValue( - transaction?.contexts?.trace.data?.method || - transaction?.request?.method || - "", - ); - return method; -} - -export function getRootTransactionName(trace: Trace) { - const method = getRootTransactionMethod(trace); - // For orphan traces without a root, use the first transaction's name - let name = trace.rootTransactionName; - if (!trace.rootTransaction && trace.transactions.length > 0) { - name = trace.transactions[0].transaction || trace.rootTransactionName; - } - - // Strip method prefix if present - const finalName = - method && name.startsWith(method) ? name.slice(method.length + 1) : name; - - return finalName; -} From c838e2b9e6613c1a9f6417c4b8f7ec5b84fabefe Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 13:41:48 +0530 Subject: [PATCH 07/15] fix(local): Count only pending ingestion admissions Let the pipeline account for admitted writes and reserve facade slots only for writes deferred behind retention controls. Immediate validation failures no longer cause false queue-full responses. Co-Authored-By: Codex --- .../src/commands/local/telemetry-buffer.ts | 43 +++++++----- .../commands/local/telemetry-buffer.test.ts | 70 +++++++++++++++++++ 2 files changed, 95 insertions(+), 18 deletions(-) diff --git a/packages/cli/src/commands/local/telemetry-buffer.ts b/packages/cli/src/commands/local/telemetry-buffer.ts index 100fbe174..a452ccb79 100644 --- a/packages/cli/src/commands/local/telemetry-buffer.ts +++ b/packages/cli/src/commands/local/telemetry-buffer.ts @@ -97,6 +97,7 @@ export function createLocalTelemetryBuffer( let controls = 0; let barrier: Promise = Promise.resolve(); const writes = new Set>(); + const deferredWrites = new Set>(); function ignored(reason: IgnoreReason): IngestResult { return { status: "ignored", @@ -118,6 +119,7 @@ export function createLocalTelemetryBuffer( // Pending controls remain ordered; ingress from the previous generation is cancelled. generation++; writes.clear(); + deferredWrites.clear(); pipeline.reset(); } const stream = (id: string) => { @@ -172,29 +174,34 @@ export function createLocalTelemetryBuffer( pipeline, ingestPayload(payload) { if (disposed) return Promise.resolve(ignored("disposed")); - if (writes.size >= maxPending) + const deferred = controls > 0; + // The pipeline counts admitted work; only writes held behind controls need extra slots. + if ( + deferred && + pipeline.getStats().pending + deferredWrites.size >= maxPending + ) return Promise.resolve(ignored("queue_full")); const epoch = generation; // A control operation blocks new admissions until its policy check and update commit. // Copy deferred inputs now, matching standalone SDK ingestion semantics. - const captured = - controls > 0 - ? { - ...payload, - body: types.isUint8Array(payload.body) - ? Buffer.from(payload.body) - : payload.body, - } - : payload; - const operation = - controls === 0 - ? pipeline.ingestPayload(captured) - : barrier.then(() => { - if (disposed) return ignored("disposed"); - if (epoch !== generation) return ignored("reset"); - return pipeline.ingestPayload(captured); - }); + const captured = deferred + ? { + ...payload, + body: types.isUint8Array(payload.body) + ? Buffer.from(payload.body) + : payload.body, + } + : payload; + const operation = !deferred + ? pipeline.ingestPayload(captured) + : barrier.then(() => { + deferredWrites.delete(operation); + if (disposed) return ignored("disposed"); + if (epoch !== generation) return ignored("reset"); + return pipeline.ingestPayload(captured); + }); writes.add(operation); + if (deferred) deferredWrites.add(operation); const release = () => { writes.delete(operation); }; diff --git a/packages/cli/test/commands/local/telemetry-buffer.test.ts b/packages/cli/test/commands/local/telemetry-buffer.test.ts index 16da1962e..dfeb801d4 100644 --- a/packages/cli/test/commands/local/telemetry-buffer.test.ts +++ b/packages/cli/test/commands/local/telemetry-buffer.test.ts @@ -528,3 +528,73 @@ test("copies cross-realm payload bytes before deferring them behind retention co buffer.dispose(); } }); + +test.each([ + { + contentType: "application/json", + status: "ignored", + reason: "unsupported_content_type", + }, + { encoding: "unsupported", status: "invalid", reason: undefined }, +])( + "permanent $status rejects do not consume admission capacity", + async ({ status, reason, ...metadata }) => { + const buffer = createLocalTelemetryBuffer(); + try { + const rejects = Array.from({ length: 100 }, () => + buffer.ingestPayload({ body: event("rejected"), ...metadata }), + ); + // A retention control must not turn already-rejected requests into pending admissions either. + const control = buffer.withWriteBarrier(async () => {}); + const fresh = buffer.ingestPayload({ body: event("fresh") }); + for (const receipt of await Promise.all(rejects)) { + expect(receipt).toMatchObject({ + status, + ...(reason ? { reason } : {}), + }); + } + await control; + expect(await fresh).toMatchObject({ status: "processed" }); + expect(buffer.read({ all: true })).toHaveLength(1); + } finally { + buffer.dispose(); + } + }, +); + +test.each([0, 32])( + "retention controls bound deferred admissions with %i pipeline writes", + async (active) => { + const buffer = createLocalTelemetryBuffer(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + try { + const running = Array.from({ length: active }, (_, index) => + buffer.ingestPayload({ body: event(`running-${index}`) }), + ); + const control = buffer.withWriteBarrier(() => gate); + const admitted = Array.from({ length: 64 - active }, (_, index) => + buffer.ingestPayload({ body: event(`deferred-${index}`) }), + ); + expect( + await buffer.ingestPayload({ body: event("overflow") }), + ).toMatchObject({ + status: "ignored", + reason: "queue_full", + }); + release(); + await control; + expect( + (await Promise.all([...running, ...admitted])).every( + (result) => result.status === "processed", + ), + ).toBe(true); + expect(buffer.read({ all: true })).toHaveLength(64); + } finally { + release(); + buffer.dispose(); + } + }, +); From 183b8083d6e4c246b4e3e42f68a6997ab54c243e Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 14:32:42 +0530 Subject: [PATCH 08/15] fix(spotlight): Upgrade malformed AI prompt fallbacks Share message normalization between the GenAI and Vercel handlers. Keep raw fallbacks replaceable until the first valid conversation is found. Co-Authored-By: Codex --- .../src/telemetry/ai-wire-values.spec.ts | 54 ++++++++++++++ packages/spotlight/src/telemetry/ai/genAI.ts | 73 +++++++------------ .../spotlight/src/telemetry/ai/messages.ts | 41 +++++++++++ .../spotlight/src/telemetry/ai/vercelAISDK.ts | 48 +----------- 4 files changed, 124 insertions(+), 92 deletions(-) create mode 100644 packages/spotlight/src/telemetry/ai/messages.ts diff --git a/packages/spotlight/src/telemetry/ai-wire-values.spec.ts b/packages/spotlight/src/telemetry/ai-wire-values.spec.ts index 5985257b8..482d96671 100644 --- a/packages/spotlight/src/telemetry/ai-wire-values.spec.ts +++ b/packages/spotlight/src/telemetry/ai-wire-values.spec.ts @@ -302,3 +302,57 @@ test("Vercel AI queries retain token usage on leaf spans without children", asyn tokensDisplay: "12 / 3", }); }); + +test.each( + [ + null, + "null", + {}, + "{}", + "not JSON", + 42, + true, + [null], + [{ content: "no role" }], + [{ role: 7 }], + ].map((invalid) => ({ invalid })), +)( + "GenAI queries upgrade malformed messages to the first valid conversation: $invalid", + async ({ invalid }) => { + const trace = await queryTrace([ + span("root", { + "gen_ai.request.messages": invalid, + "gen_ai.system_instructions": "Be concise", + }), + span( + "valid", + { + "gen_ai.request.messages": JSON.stringify([ + { role: "user", content: [{ type: "text", text: "Structured" }] }, + ]), + }, + "root", + ), + span( + "later", + { + "gen_ai.request.messages": [ + { role: "user", content: "Later conversation" }, + ], + }, + "root", + ), + ]); + expect(trace?.prompt).toEqual({ + messages: [{ role: "user", content: "Structured" }], + system: "Be concise", + }); + }, +); + +test("GenAI queries omit null request messages", async () => { + const trace = await queryTrace([ + span("root", { "gen_ai.request.messages": null }), + ]); + expect(trace?.prompt).toBeUndefined(); +}); diff --git a/packages/spotlight/src/telemetry/ai/genAI.ts b/packages/spotlight/src/telemetry/ai/genAI.ts index 5db0fdfa9..7b52bd1bd 100644 --- a/packages/spotlight/src/telemetry/ai/genAI.ts +++ b/packages/spotlight/src/telemetry/ai/genAI.ts @@ -1,3 +1,4 @@ +import { normalizePromptMessages } from "./messages.ts"; import { stringifyValue } from "../utils/stringifyValue.ts"; import { finiteNumber } from "../utils/finiteNumber.ts"; import { getAllSpansInTree } from "../store/helpers.ts"; @@ -147,10 +148,11 @@ export const genAIHandler: AILibraryHandler = { }; const responseState = { finishReasonFromText: false }; + const promptState = { hasRequestMessages: false }; for (const span of allSpans) { if (!span.data) continue; extractMetadata(span, trace); - extractPromptData(span, trace); + extractPromptData(span, trace, promptState); extractResponseData(span, trace, responseState); extractToolCallData(span, trace); } @@ -240,24 +242,6 @@ function formatTokensDisplay( return UNKNOWN_OPERATION; } -/** - * Normalizes message content from a string or an array of content blocks to a - * plain string. Only text blocks are surfaced; images/files would need UI work. - */ -function normalizeMessageContent(content: unknown): string { - if (typeof content === "string") return content; - if (Array.isArray(content)) { - return content - .filter( - (block): block is { type: string; text: string } => - block?.type === "text" && typeof block.text === "string", - ) - .map((block) => block.text) - .join(""); - } - return ""; -} - function extractMetadata(span: Span, trace: AITrace) { if (!span.data) return; @@ -280,38 +264,35 @@ function extractMetadata(span: Span, trace: AITrace) { } } -function extractPromptData(span: Span, trace: AITrace) { - if (!span.data || trace.prompt) return; +function extractPromptData( + span: Span, + trace: AITrace, + state: { hasRequestMessages: boolean }, +) { + if (!span.data || state.hasRequestMessages) return; const promptMessages = span.data[GEN_AI_REQUEST_MESSAGES_FIELD]; - if (promptMessages === undefined) return; - const system = span.data[GEN_AI_SYSTEM_INSTRUCTIONS_FIELD]; - const systemText = system !== undefined ? stringifyValue(system) : undefined; - - try { - const messages = - typeof promptMessages === "string" - ? JSON.parse(promptMessages) - : promptMessages; - if (Array.isArray(messages)) { - const normalizedMessages = messages.map( - (msg: { role: string; content: unknown }) => ({ - ...msg, - content: normalizeMessageContent(msg.content), - }), - ); - trace.prompt = { messages: normalizedMessages, system: systemText }; - return; - } - } catch { - // fall through to the raw representation below + const systemText = + system !== undefined && system !== null + ? stringifyValue(system) + : trace.prompt?.system; + const structured = normalizePromptMessages(promptMessages); + if (structured !== undefined) { + trace.prompt = { messages: structured, system: systemText }; + state.hasRequestMessages = true; + return; } + if (trace.prompt) return; - trace.prompt = { - messages: [{ role: "unknown", content: stringifyValue(promptMessages) }], - system: systemText, - }; + if (promptMessages !== undefined && promptMessages !== null) { + trace.prompt = { + messages: [{ role: "unknown", content: stringifyValue(promptMessages) }], + system: systemText, + }; + } else if (systemText !== undefined) { + trace.prompt = { system: systemText }; + } } function extractResponseData( diff --git a/packages/spotlight/src/telemetry/ai/messages.ts b/packages/spotlight/src/telemetry/ai/messages.ts new file mode 100644 index 000000000..4edc9e54b --- /dev/null +++ b/packages/spotlight/src/telemetry/ai/messages.ts @@ -0,0 +1,41 @@ +import type { AIPrompt } from "../types.ts"; +import { parsed, record } from "../internal/wireValues.ts"; + +/** + * Normalizes message content from v1 (string) or v2 (array of content blocks) format to a string. + * v1: content = "text string" + * v2: content = [{ type: "text", text: "text string" }] + * + * Only extracts text content blocks. Other block types (image, file, tool-call) + * would require UI changes to display properly and are filtered out for now. + */ +function normalizeMessageContent(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + // Only extract text content - images/files would need UI changes to display + return content + .filter( + (block): block is { type: string; text: string } => + block?.type === "text" && typeof block.text === "string", + ) + .map((block) => block.text) + .join(""); + } + return ""; +} + +export function normalizePromptMessages(value: unknown): AIPrompt["messages"] { + const messages = parsed(value); + if (!Array.isArray(messages)) return undefined; + const normalized: NonNullable = []; + for (const value of messages) { + const message = record(value); + if (typeof message.role !== "string") return undefined; + normalized.push({ + ...message, + role: message.role, + content: normalizeMessageContent(message.content), + }); + } + return normalized; +} diff --git a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts index 0d1e69eea..eda9c021b 100644 --- a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts +++ b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts @@ -1,13 +1,8 @@ +import { normalizePromptMessages } from "./messages.ts"; import { stringifyValue } from "../utils/stringifyValue.ts"; import { finiteNumber } from "../utils/finiteNumber.ts"; import { getAllSpansInTree } from "../store/helpers.ts"; -import type { - AILibraryHandler, - AIPrompt, - AIToolCall, - AITrace, - Span, -} from "../types.ts"; +import type { AILibraryHandler, AIToolCall, AITrace, Span } from "../types.ts"; import { extractFinishReason } from "./finishReason.ts"; import { parsed, record } from "../internal/wireValues.ts"; @@ -308,45 +303,6 @@ function formatTokensDisplay( return UNKNOWN_OPERATION; } -/** - * Normalizes message content from v1 (string) or v2 (array of content blocks) format to a string. - * v1: content = "text string" - * v2: content = [{ type: "text", text: "text string" }] - * - * Only extracts text content blocks. Other block types (image, file, tool-call) - * would require UI changes to display properly and are filtered out for now. - */ -function normalizeMessageContent(content: unknown): string { - if (typeof content === "string") return content; - if (Array.isArray(content)) { - // Only extract text content - images/files would need UI changes to display - return content - .filter( - (block): block is { type: string; text: string } => - block?.type === "text" && typeof block.text === "string", - ) - .map((block) => block.text) - .join(""); - } - return ""; -} - -function normalizePromptMessages(value: unknown): AIPrompt["messages"] { - const messages = parsed(value); - if (!Array.isArray(messages)) return undefined; - const normalized: NonNullable = []; - for (const value of messages) { - const message = record(value); - if (typeof message.role !== "string") return undefined; - normalized.push({ - ...message, - role: message.role, - content: normalizeMessageContent(message.content), - }); - } - return normalized; -} - function parseSpanData(spans: Span[], trace: AITrace) { const promptState = { hasRequestMessages: false }; const responseState = { finishReasonFromText: false }; From 56066f0d7c1dd63f54271d479e10d2a06a3162f2 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 15:26:38 +0530 Subject: [PATCH 09/15] fix(spotlight): Read standard Vercel response tool calls Read standard tool-call attributes when the Vercel alias is absent or malformed. Preserve valid proprietary values, including explicit empty arrays. Co-Authored-By: Codex --- .../src/telemetry/ai-wire-values.spec.ts | 45 +++++++++++++++++++ .../spotlight/src/telemetry/ai/vercelAISDK.ts | 19 ++++---- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/packages/spotlight/src/telemetry/ai-wire-values.spec.ts b/packages/spotlight/src/telemetry/ai-wire-values.spec.ts index 482d96671..3617e801c 100644 --- a/packages/spotlight/src/telemetry/ai-wire-values.spec.ts +++ b/packages/spotlight/src/telemetry/ai-wire-values.spec.ts @@ -356,3 +356,48 @@ test("GenAI queries omit null request messages", async () => { ]); expect(trace?.prompt).toBeUndefined(); }); + +test.each( + [calls, JSON.stringify(calls), []].map((toolCalls) => ({ toolCalls })), +)( + "Vercel AI queries read standard response tool calls on child spans: $toolCalls", + async ({ toolCalls }) => { + const trace = await queryTrace([ + span("root", { "vercel.ai.operationId": "ai.generateText" }), + span("child", { "gen_ai.response.tool_calls": toolCalls }, "root"), + ]); + expect(trace?.response?.toolCalls).toEqual( + typeof toolCalls === "string" ? calls : toolCalls, + ); + }, +); + +test.each(invalidCalls.map((invalid) => ({ invalid })))( + "Vercel AI queries fall back to standard tool calls after a malformed proprietary alias: $invalid", + async ({ invalid }) => { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.response.toolCalls": invalid, + "gen_ai.response.tool_calls": JSON.stringify(calls), + }), + ]); + expect(trace?.response?.toolCalls).toEqual(calls); + }, +); + +test.each([calls, []].map((toolCalls) => ({ toolCalls })))( + "Vercel AI queries keep a valid proprietary tool-call alias ahead of the standard fallback: $toolCalls", + async ({ toolCalls }) => { + const trace = await queryTrace([ + span("root", { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.response.toolCalls": toolCalls, + "gen_ai.response.tool_calls": [ + { toolCallId: "other", toolName: "other", args: {} }, + ], + }), + ]); + expect(trace?.response?.toolCalls).toEqual(toolCalls); + }, +); diff --git a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts index eda9c021b..7f85f2ee8 100644 --- a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts +++ b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts @@ -29,6 +29,7 @@ const AI_PROMPT_FIELD = "vercel.ai.prompt"; const AI_PROMPT_MESSAGES_FIELD = "gen_ai.request.messages"; const AI_RESPONSE_FINISH_REASON_FIELD = "vercel.ai.response.finishReason"; const GEN_AI_RESPONSE_FINISH_REASONS_FIELD = "gen_ai.response.finish_reasons"; +const GEN_AI_RESPONSE_TOOL_CALLS_FIELD = "gen_ai.response.tool_calls"; const AI_RESPONSE_TEXT_FIELD = "gen_ai.response.text"; const AI_RESPONSE_TOOL_CALLS_FIELD = "vercel.ai.response.toolCalls"; @@ -428,16 +429,14 @@ function extractResponseData( trace.response.finishReason = finishReason; } - const toolCalls = span.data[AI_RESPONSE_TOOL_CALLS_FIELD]; - if (toolCalls) { - try { - const parsed: unknown = - typeof toolCalls === "string" ? JSON.parse(toolCalls) : toolCalls; - if (Array.isArray(parsed)) { - trace.response.toolCalls = parsed; - } - } catch { - // Truncated tool-call JSON must not discard the rest of the AI trace. + for (const field of [ + AI_RESPONSE_TOOL_CALLS_FIELD, + GEN_AI_RESPONSE_TOOL_CALLS_FIELD, + ]) { + const toolCalls = parsed(span.data[field]); + if (Array.isArray(toolCalls)) { + trace.response.toolCalls = toolCalls; + break; } } } From 9d0498fad6e29f1412ddb5f49ed16a839d9a015d Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 17:21:26 +0530 Subject: [PATCH 10/15] fix(spotlight): Bound orphan spans to their children Extend synthetic parent time ranges as later siblings are attached. Avoid overlapping profile gaps while preserving real children and trailing samples. Co-Authored-By: Codex --- .../src/telemetry/data/profiles.test.ts | 66 +++++++++++++++++++ .../spotlight/src/telemetry/utils/traces.ts | 9 +++ 2 files changed, 75 insertions(+) diff --git a/packages/spotlight/src/telemetry/data/profiles.test.ts b/packages/spotlight/src/telemetry/data/profiles.test.ts index d4c833edc..8d29ecbca 100644 --- a/packages/spotlight/src/telemetry/data/profiles.test.ts +++ b/packages/spotlight/src/telemetry/data/profiles.test.ts @@ -339,3 +339,69 @@ describe("profile graft thread selection", () => { ).toEqual(["main", "worker"]); }); }); + +describe("profile graft missing parents", () => { + it.each([undefined, 20])( + "grafts each orphan-child gap once with outer work ending at %s", + (rootEnd) => { + const target = trace(); + if (rootEnd !== undefined) { + target.spans.set("root", { + span_id: "root", + start_timestamp: 0, + timestamp: rootEnd, + }); + } + target.spans.set("first", { + span_id: "first", + parent_span_id: "missing", + start_timestamp: 0, + timestamp: 5, + }); + target.spans.set("second", { + span_id: "second", + parent_span_id: "missing", + start_timestamp: 10, + timestamp: 15, + }); + const source = profile({ + samples: [0, 5, 10, 15].map((start_timestamp) => ({ + thread_id: "main", + stack_id: 0, + start_timestamp, + })), + }); + const intervals = () => + Array.from(target.spans.values()) + .filter( + (span) => span.tags?.source === "profile" && span.op === "main", + ) + .map((span) => [span.start_timestamp, span.timestamp]) + .sort((a, b) => a[0] - b[0]); + + graftProfileSpans(target, source); + + // Real leaves, the gap between them, and any outer tail each own their samples. + const expected = [ + [0, 5], + [5, 10], + [10, 15], + ]; + if (rootEnd !== undefined) expected.push([15, 20]); + expect(intervals()).toEqual(expected); + expect(target.spans.get("missing")).toMatchObject({ + start_timestamp: 0, + timestamp: 15, + children: [ + { span_id: "first", parent_span_id: "missing" }, + { tags: { source: "profile" }, start_timestamp: 5, timestamp: 10 }, + { span_id: "second", parent_span_id: "missing" }, + ], + }); + const ids = Array.from(target.spans.keys()); + graftProfileSpans(target, source); + expect(intervals()).toEqual(expected); + expect(Array.from(target.spans.keys())).toEqual(ids); + }, + ); +}); diff --git a/packages/spotlight/src/telemetry/utils/traces.ts b/packages/spotlight/src/telemetry/utils/traces.ts index 81b4ce1eb..deea60a3e 100644 --- a/packages/spotlight/src/telemetry/utils/traces.ts +++ b/packages/spotlight/src/telemetry/utils/traces.ts @@ -17,6 +17,15 @@ export function groupSpans(spans: Map): Span[] { if (parent) { parent.children ??= []; parent.children.push(span); + if (parent.op === "orphan") { + // Missing-parent wrappers must cover every child so profile gaps are + // assigned either inside the wrapper or after it, never both. + parent.start_timestamp = Math.min( + parent.start_timestamp, + span.start_timestamp, + ); + parent.timestamp = Math.max(parent.timestamp, span.timestamp); + } } else if (span.parent_span_id && span.parent_span_id !== span.span_id) { const parentParent = sortedSpans.find((s) => !s.parent_span_id); parent = { From ac75cbcac99a5edfdf3bff8ce41e7d33731c5cbc Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 17:24:46 +0530 Subject: [PATCH 11/15] fix(spotlight): Preserve structured SDK report envelopes Classify known JSON wire items separately from processed event types. Keep report fields available to subscribers and validate malformed payloads. Co-Authored-By: Codex --- packages/spotlight/src/sdk/node.spec.ts | 71 ++++++++++++++++++- .../src/telemetry/envelope/codec.spec.ts | 42 +++++++++++ .../spotlight/src/telemetry/envelope/codec.ts | 13 +++- 3 files changed, 120 insertions(+), 6 deletions(-) diff --git a/packages/spotlight/src/sdk/node.spec.ts b/packages/spotlight/src/sdk/node.spec.ts index 674b3318a..b68c44d7a 100644 --- a/packages/spotlight/src/sdk/node.spec.ts +++ b/packages/spotlight/src/sdk/node.spec.ts @@ -1,4 +1,5 @@ // @vitest-environment node +import { Buffer } from "node:buffer"; import { readFileSync } from "node:fs"; import { runInNewContext } from "node:vm"; import { brotliCompressSync, deflateSync, gzipSync } from "node:zlib"; @@ -10,6 +11,70 @@ const fixture = readFileSync( ); describe("Node Telemetry adapter", () => { + test.each([ + [ + "client_report", + { discarded_events: [{ reason: "sample_rate", quantity: 1 }] }, + ], + ["session", { sid: "session-id", status: "ok", attrs: { release: "1.0" } }], + [ + "user_report", + { event_id: "reported-event", comments: "Something broke" }, + ], + ] as const)( + "publishes retained %s JSON fields with an envelope-only receipt", + async (type, payload) => { + const telemetry = createTelemetryPipeline(); + try { + const body = Buffer.from( + `{}\n${JSON.stringify({ type })}\n${JSON.stringify(payload)}`, + ); + const observed: unknown[] = []; + telemetry.subscribe((snapshot) => + observed.push(snapshot.envelopes[0][1][0][1]), + ); + const result = await telemetry.ingestPayload({ body }); + expect(result).toMatchObject({ + status: "ignored", + reason: "unsupported_items", + envelopeId: expect.any(String), + }); + if (result.status !== "ignored" || !result.envelopeId) + throw new Error("Expected retained envelope-only receipt"); + expect(observed).toEqual([{ ...payload, type }]); + expect(telemetry.getEnvelopes()[0][1][0][1]).toEqual({ + ...payload, + type, + }); + expect(telemetry.getDecodedPayload(result.envelopeId)?.body).toEqual( + body, + ); + } finally { + telemetry.dispose(); + } + }, + ); + + test.each(["client_report", "session", "user_report"])( + "rejects malformed %s JSON instead of retaining it as opaque bytes", + async (type) => { + const telemetry = createTelemetryPipeline(); + try { + expect( + await telemetry.ingestPayload({ + body: `{}\n${JSON.stringify({ type })}\nnot json`, + }), + ).toMatchObject({ + status: "invalid", + diagnostics: [{ code: "invalid_envelope" }], + }); + expect(telemetry.getEnvelopes()).toEqual([]); + } finally { + telemetry.dispose(); + } + }, + ); + test.each(["replay_recording", "future_binary"])( "retains an opaque %s item alongside supported telemetry", async (type) => { @@ -70,7 +135,7 @@ describe("Node Telemetry adapter", () => { "subscribers observe %s raw bytes together with the envelope in one notification", async (_, body) => { const spotlight = createTelemetryPipeline(); - const observations: (Buffer | undefined)[] = []; + const observations: (Uint8Array | undefined)[] = []; spotlight.subscribe((snapshot) => { const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); observations.push(spotlight.getDecodedPayload(id)?.body); @@ -88,8 +153,8 @@ describe("Node Telemetry adapter", () => { if (first.status !== "processed") throw new Error("Expected processed envelope"); const observations: { - old: Buffer | undefined; - newest: Buffer | undefined; + old: Uint8Array | undefined; + newest: Uint8Array | undefined; }[] = []; spotlight.subscribe((snapshot) => { const envelope = snapshot.envelopes.find( diff --git a/packages/spotlight/src/telemetry/envelope/codec.spec.ts b/packages/spotlight/src/telemetry/envelope/codec.spec.ts index 0a76814db..626b3d27d 100644 --- a/packages/spotlight/src/telemetry/envelope/codec.spec.ts +++ b/packages/spotlight/src/telemetry/envelope/codec.spec.ts @@ -6,6 +6,48 @@ import { decodeEnvelope } from "./codec.ts"; const encode = (value: string) => new TextEncoder().encode(value); describe("portable byte envelope codec", () => { + test.each([ + [ + "client_report", + { discarded_events: [{ reason: "sample_rate", quantity: 1 }] }, + ], + ["session", { sid: "session-id", status: "ok", attrs: { release: "1.0" } }], + [ + "user_report", + { event_id: "reported-event", comments: "Something broke" }, + ], + ] as const)( + "strict decoding retains %s JSON fields without a content type", + (type, payload) => { + const bytes = encode( + `{}\n${JSON.stringify({ type })}\n${JSON.stringify(payload)}`, + ); + const errors = vi.fn(); + const strict = decodeEnvelope(bytes, undefined, errors, true)!; + expect(strict[1][0][1]).toEqual({ ...payload, type }); + expect(strict[1]).toEqual(decodeEnvelope(bytes)![1]); + expect(errors).not.toHaveBeenCalled(); + }, + ); + + test.each(["future_json", "replay_recording"])( + "strict decoding keeps unrecognized %s payloads opaque", + (type) => { + const payload = encode('{"custom":"value"}\nopaque'); + const header = encode( + `{}\n${JSON.stringify({ type, length: payload.length })}\n`, + ); + const bytes = new Uint8Array(header.length + payload.length); + bytes.set(header); + bytes.set(payload, header.length); + const errors = vi.fn(); + expect(decodeEnvelope(bytes, undefined, errors, true)![1][0][1]).toEqual({ + data: btoa(String.fromCharCode(...payload)), + }); + expect(errors).not.toHaveBeenCalled(); + }, + ); + test("parses Uint8Array text attachment bytes with explicit byte lengths", () => { const text = "é\nhello"; const payload = encode(text); diff --git a/packages/spotlight/src/telemetry/envelope/codec.ts b/packages/spotlight/src/telemetry/envelope/codec.ts index 673c558c2..914c6d228 100644 --- a/packages/spotlight/src/telemetry/envelope/codec.ts +++ b/packages/spotlight/src/telemetry/envelope/codec.ts @@ -4,6 +4,15 @@ import { RAW_TYPES, SUPPORTED_EVENT_TYPES } from "../constants/sentry.ts"; import { opaquePayload } from "./payload.ts"; import { type SourceType, inferEnvelopeSource } from "./source.ts"; +// JSON wire items also include reports retained for inspection without a processed model. +const JSON_ITEM_TYPES = new Set([ + ...SUPPORTED_EVENT_TYPES, + "span", + "client_report", + "session", + "user_report", +]); + /** * Spotlight-specific extensions added to envelope headers for internal tracking */ @@ -104,9 +113,7 @@ export function decodeEnvelope( try { if ( RAW_TYPES.has(itemHeader.type) || - (strict && - !SUPPORTED_EVENT_TYPES.has(itemHeader.type) && - itemHeader.type !== "span") + (strict && !JSON_ITEM_TYPES.has(itemHeader.type)) ) { if ( !itemHeader.content_type && From b038bdc3c4f5dc69a4ad209c353a87c913bb2da0 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 18:37:01 +0530 Subject: [PATCH 12/15] fix(local): Sort promoted waterfall spans chronologically Re-sort visible siblings after hiding missing-parent wrappers. Preserve captured parent identifiers and keep the SDK span tree immutable. Co-Authored-By: Codex --- apps/local/src/lib/trace-model.test.ts | 58 ++++++++++++++++++++++++++ apps/local/src/lib/trace-model.ts | 6 ++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/apps/local/src/lib/trace-model.test.ts b/apps/local/src/lib/trace-model.test.ts index f2cfbcdf3..848b7a71b 100644 --- a/apps/local/src/lib/trace-model.test.ts +++ b/apps/local/src/lib/trace-model.test.ts @@ -357,6 +357,64 @@ describe("buildTraceGroups", () => { ]); }); + test.each([true, false])( + "orders promoted missing-parent spans with their visible siblings (real root: %s)", + async (hasRoot) => { + const [trace] = await buildTraceGroups( + decodeEnvelope( + JSON.stringify([ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + ...(hasRoot + ? [{ span_id: "root", start_timestamp: 0 }] + : []), + { + span_id: "a", + parent_span_id: "missing-parent", + start_timestamp: 1, + }, + { + span_id: "b", + parent_span_id: hasRoot ? "root" : "other-missing-parent", + start_timestamp: 2, + }, + { + span_id: "c", + parent_span_id: "missing-parent", + start_timestamp: 3, + }, + ].map((span) => ({ + ...span, + trace_id: "trace-promoted-order", + end_timestamp: 4, + attributes: { + "sentry.op": { type: "string", value: "task" }, + }, + })), + }, + ], + ], + ]), + "promoted-order", + ), + ); + + if (hasRoot) expect(trace?.roots).toHaveLength(1); + const siblings = hasRoot ? trace?.roots[0]?.children : trace?.roots; + expect(siblings?.map((span) => span.id)).toEqual(["a", "b", "c"]); + expect(siblings?.map((span) => span.parentId)).toEqual([ + "missing-parent", + hasRoot ? "root" : "other-missing-parent", + "missing-parent", + ]); + }, + ); + test("nests a standalone span using its top-level parent span id", async () => { const traces = await buildTraceGroups([ item({ diff --git a/apps/local/src/lib/trace-model.ts b/apps/local/src/lib/trace-model.ts index 498c8445a..caf9666a7 100644 --- a/apps/local/src/lib/trace-model.ts +++ b/apps/local/src/lib/trace-model.ts @@ -128,7 +128,11 @@ export function adaptTraceGroups( visible.push(adapt(span)); } } - return visible; + // Promoted children can interleave with siblings outside their wrapper. + return visible.sort( + (a, b) => + (a.startTimestamp ?? Infinity) - (b.startTimestamp ?? Infinity), + ); } for (const span of sharedSpans) { const local = mapped.get(span.span_id); From de8a1a38049a19108abf70123b9f795e616c0b3b Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 19:34:50 +0530 Subject: [PATCH 13/15] fix(spotlight): Preserve usable AI inspection fallbacks Read nested system instructions and prefer structured conversations and tool-call arrays over malformed aliases while retaining raw inspection values. Treat null model and provider attributes as absent so later spans can supply metadata. Co-authored-by: Codex --- .../src/telemetry/ai-inspection.spec.ts | 120 ++++++++++++++++++ .../src/telemetry/ai-wire-values.spec.ts | 65 ++++++++++ packages/spotlight/src/telemetry/ai/genAI.ts | 5 +- .../spotlight/src/telemetry/ai/inspection.ts | 56 +++++--- .../spotlight/src/telemetry/ai/vercelAISDK.ts | 2 +- 5 files changed, 226 insertions(+), 22 deletions(-) create mode 100644 packages/spotlight/src/telemetry/ai-inspection.spec.ts diff --git a/packages/spotlight/src/telemetry/ai-inspection.spec.ts b/packages/spotlight/src/telemetry/ai-inspection.spec.ts new file mode 100644 index 000000000..1549c3d4c --- /dev/null +++ b/packages/spotlight/src/telemetry/ai-inspection.spec.ts @@ -0,0 +1,120 @@ +import { expect, test } from "vitest"; +import { inspectAI } from "../sdk/core.ts"; + +const messages = [ + { + role: "user", + content: [ + { type: "text", text: "Describe the image" }, + { type: "image", image: "https://example.com/image.png" }, + ], + tool_calls: [{ id: "call", function: { name: "image", arguments: "{}" } }], + }, +]; + +test.each(["gen_ai.prompt", "vercel.ai.prompt", "ai.prompt"])( + "AI inspection exposes system instructions in object and JSON %s wrappers", + (field) => { + const wrapper = { system: "Be concise", messages }; + for (const value of [wrapper, JSON.stringify(wrapper)]) { + const payload = { data: { [field]: value } }; + const before = structuredClone(payload); + expect(inspectAI(payload)).toMatchObject({ + system: "Be concise", + messages: wrapper, + }); + expect(payload).toEqual(before); + expect( + inspectAI({ + data: { + [field]: value, + "gen_ai.request.messages": messages, + "gen_ai.system_instructions": "Explicit instructions", + }, + }), + ).toMatchObject({ system: "Explicit instructions", messages }); + } + }, +); + +test.each([ + "gen_ai.input.messages", + "vercel.ai.prompt.messages", + "vercel.ai.prompt", + "gen_ai.prompt", + "ai.prompt.messages", + "ai.prompt", +])( + "AI inspection selects a usable %s after malformed request messages without normalizing content", + (field) => { + const fallback = field.endsWith(".messages") + ? messages + : { system: "Nested instructions", messages }; + for (const invalid of ["null", {}, "not JSON", [null]]) { + const payload = { + attributes: { + "gen_ai.request.messages": { value: invalid }, + [field]: { value: JSON.stringify(fallback) }, + }, + }; + const before = structuredClone(payload); + expect(inspectAI(payload)?.messages).toEqual(fallback); + expect(payload).toEqual(before); + } + }, +); + +test.each([ + { value: "null", expected: null }, + { value: "{}", expected: {} }, + { value: "not JSON", expected: "not JSON" }, + { value: [{ content: "No role" }], expected: [{ content: "No role" }] }, +])( + "AI inspection retains the first raw malformed prompt when no usable conversation exists: $value", + ({ value, expected }) => { + expect( + inspectAI({ + data: { + "gen_ai.request.messages": value, + "gen_ai.prompt": { unexpected: "retained" }, + }, + })?.messages, + ).toEqual(expected); + }, +); + +test("AI inspection preserves explicit structured message precedence and nested system fallback", () => { + expect( + inspectAI({ + data: { + "gen_ai.request.messages": messages, + "vercel.ai.prompt": { system: "Nested instructions", messages: [] }, + }, + }), + ).toMatchObject({ messages, system: "Nested instructions" }); +}); + +test.each(["vercel.ai.response.toolCalls", "ai.response.toolCalls"])( + "AI inspection prefers usable %s after malformed standard tool calls", + (field) => { + const calls = [ + { id: "call", name: "weather", arguments: { city: "Pune" } }, + ]; + for (const invalid of ["null", {}, "not JSON"]) { + expect( + inspectAI({ + data: { + "gen_ai.response.tool_calls": invalid, + [field]: JSON.stringify(calls), + }, + })?.tools, + ).toEqual(calls); + } + }, +); + +test("AI inspection keeps raw tool calls when no array alias is captured", () => { + expect( + inspectAI({ data: { "gen_ai.response.tool_calls": "not JSON" } })?.tools, + ).toEqual(["not JSON"]); +}); diff --git a/packages/spotlight/src/telemetry/ai-wire-values.spec.ts b/packages/spotlight/src/telemetry/ai-wire-values.spec.ts index 3617e801c..cc51ca4b1 100644 --- a/packages/spotlight/src/telemetry/ai-wire-values.spec.ts +++ b/packages/spotlight/src/telemetry/ai-wire-values.spec.ts @@ -56,6 +56,71 @@ const calls = [ { toolCallId: "weather-call", toolName: "weather", args: { city: "Pune" } }, ]; +test.each(["generic", "vercel"])( + "%s AI queries treat null root model and provider metadata as absent for child fallback", + async (provider) => { + const trace = await queryTrace([ + span("root", { + ...(provider === "vercel" + ? { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.model.id": null, + "vercel.ai.model.provider": null, + } + : {}), + "gen_ai.request.model": null, + "gen_ai.response.model": null, + "gen_ai.system": null, + }), + span( + "child", + { + "gen_ai.response.model": "child-model", + "gen_ai.system": "child-provider", + }, + "root", + ), + ]); + expect(trace?.metadata).toMatchObject({ + modelId: "child-model", + modelProvider: "child-provider", + }); + }, +); + +test.each(["generic", "vercel"])( + "%s AI queries omit null-only metadata and preserve same-span model aliases", + async (provider) => { + const data = { + ...(provider === "vercel" + ? { + "vercel.ai.operationId": "ai.generateText", + "vercel.ai.model.id": null, + "vercel.ai.model.provider": null, + } + : {}), + "gen_ai.request.model": null, + "gen_ai.response.model": null, + "gen_ai.system": null, + }; + const missing = await queryTrace([span("root", data)]); + expect(missing?.metadata.modelId).toBeUndefined(); + expect(missing?.metadata.modelProvider).toBeUndefined(); + + const fallback = await queryTrace([ + span("root", { + ...data, + "gen_ai.response.model": "response-model", + "gen_ai.system": "response-provider", + }), + ]); + expect(fallback?.metadata).toMatchObject({ + modelId: "response-model", + modelProvider: "response-provider", + }); + }, +); + test.each(["null", "{}", "not JSON"])( "Vercel AI queries preserve a same-span wrapper after malformed request messages: %s", async (invalid) => { diff --git a/packages/spotlight/src/telemetry/ai/genAI.ts b/packages/spotlight/src/telemetry/ai/genAI.ts index 7b52bd1bd..daa5b4a04 100644 --- a/packages/spotlight/src/telemetry/ai/genAI.ts +++ b/packages/spotlight/src/telemetry/ai/genAI.ts @@ -249,14 +249,15 @@ function extractMetadata(span: Span, trace: AITrace) { const modelId = span.data[GEN_AI_REQUEST_MODEL_FIELD] ?? span.data[GEN_AI_RESPONSE_MODEL_FIELD]; - if (modelId !== undefined) { + if (modelId !== undefined && modelId !== null) { trace.metadata.modelId = stringifyValue(modelId); } } if ( trace.metadata.modelProvider === undefined && - span.data[GEN_AI_SYSTEM_FIELD] !== undefined + span.data[GEN_AI_SYSTEM_FIELD] !== undefined && + span.data[GEN_AI_SYSTEM_FIELD] !== null ) { trace.metadata.modelProvider = stringifyValue( span.data[GEN_AI_SYSTEM_FIELD], diff --git a/packages/spotlight/src/telemetry/ai/inspection.ts b/packages/spotlight/src/telemetry/ai/inspection.ts index 91d96fd52..b1a26c023 100644 --- a/packages/spotlight/src/telemetry/ai/inspection.ts +++ b/packages/spotlight/src/telemetry/ai/inspection.ts @@ -39,6 +39,14 @@ export type AIInspection = { response?: unknown; tools: unknown[]; }; + +function hasMessages(value: unknown): boolean { + return ( + Array.isArray(value) && + value.every((message) => typeof record(message).role === "string") + ); +} + export function inspectAI(payload: unknown): AIInspection | undefined { if (!isAIInteraction(payload)) return; const data = readSpanData(payload); @@ -79,13 +87,14 @@ export function inspectAI(payload: unknown): AIInspection | undefined { ["maxRetries", ["vercel.ai.settings.maxRetries"]], ["maxSteps", ["vercel.ai.settings.maxSteps"]], ]; - const calls = parsed( - pick( - "gen_ai.response.tool_calls", - "vercel.ai.response.toolCalls", - "ai.response.toolCalls", - ), - ); + const callKeys = [ + "gen_ai.response.tool_calls", + "vercel.ai.response.toolCalls", + "ai.response.toolCalls", + ]; + const calls = + callKeys.map((key) => parsed(data[key])).find(Array.isArray) ?? + parsed(pick(...callKeys)); const tools = Array.isArray(calls) ? [...calls] : calls === undefined @@ -99,6 +108,25 @@ export function inspectAI(payload: unknown): AIInspection | undefined { arguments: parsed(pick("gen_ai.tool.input", "ai.toolCall.args")), result: parsed(pick("gen_ai.tool.output", "ai.toolCall.result")), }); + const messageKeys = [ + "gen_ai.request.messages", + "gen_ai.input.messages", + "vercel.ai.prompt.messages", + "vercel.ai.prompt", + "gen_ai.prompt", + "ai.prompt.messages", + "ai.prompt", + ]; + const prompts = messageKeys.map((key) => parsed(data[key])); + // Validate the conversation shape without normalizing its content: inspection + // must retain image blocks, tool calls, and unknown captured fields. + const messages = + prompts.find( + (value) => hasMessages(value) || hasMessages(record(value).messages), + ) ?? parsed(pick(...messageKeys)); + const wrappedSystem = prompts + .map((value) => record(value).system) + .find((value) => value !== undefined && value !== null); return { fields: Object.fromEntries( definitions.flatMap(([label, keys]) => { @@ -111,19 +139,9 @@ export function inspectAI(payload: unknown): AIInspection | undefined { "gen_ai.system_instructions", "gen_ai.prompt.system", "ai.prompt.system", - ), - ), - messages: parsed( - pick( - "gen_ai.request.messages", - "gen_ai.input.messages", - "vercel.ai.prompt.messages", - "vercel.ai.prompt", - "gen_ai.prompt", - "ai.prompt.messages", - "ai.prompt", - ), + ) ?? wrappedSystem, ), + messages, response: parsed( pick( "gen_ai.response.text", diff --git a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts index 7f85f2ee8..e97a033f3 100644 --- a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts +++ b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts @@ -100,7 +100,7 @@ function extractFieldFromSpan( ): unknown { const fields = AI_FIELD_MAPPINGS[fieldKey]; for (const field of fields) { - if (data[field] !== undefined) { + if (data[field] !== undefined && data[field] !== null) { return data[field]; } } From 35d898a8f51a91e0d84fa5ba772f4df2d70716ff Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 19:35:09 +0530 Subject: [PATCH 14/15] fix(spotlight): Keep consolidated profile parent links valid Reparent merged children and lifted frames to their surviving parent spans. Co-authored-by: Codex --- .../spotlight/src/telemetry/data/profiles.ts | 18 ++- .../src/telemetry/profile-depth.spec.ts | 17 ++- .../telemetry/profile-parent-links.spec.ts | 134 ++++++++++++++++++ 3 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 packages/spotlight/src/telemetry/profile-parent-links.spec.ts diff --git a/packages/spotlight/src/telemetry/data/profiles.ts b/packages/spotlight/src/telemetry/data/profiles.ts index 7575046df..dc90e0553 100644 --- a/packages/spotlight/src/telemetry/data/profiles.ts +++ b/packages/spotlight/src/telemetry/data/profiles.ts @@ -58,6 +58,10 @@ function consolidateSpans(trace: Trace, spans: Span[]): Span[] { // Require the spans are sorted by start_timestamp lastSpan.timestamp = span.timestamp; if (span.children) { + // This sample's span is discarded; its children now belong to the survivor. + for (const child of span.children) { + child.parent_span_id = lastSpan.span_id; + } if (lastSpan.children) { for (const child of span.children) { lastSpan.children.push(child); @@ -234,11 +238,17 @@ export function getSpansFromProfile( if (!fillerSpans.length) { return []; } - const consolidated = consolidateSpans(trace, fillerSpans); + let consolidated = consolidateSpans(trace, fillerSpans); // Remove the extra layer of nesting if there is only one span which should be the "Thread" span - return ( - consolidated.length === 1 ? consolidated[0].children || [] : consolidated - ).filter( + if (consolidated.length === 1) { + const threadSpan = consolidated[0]; + consolidated = threadSpan.children || []; + trace.spans.delete(threadSpan.span_id); + for (const child of consolidated) { + child.parent_span_id = parent_span_id; + } + } + return consolidated.filter( (span) => span.timestamp - span.start_timestamp > 0 && span.timestamp <= endTs, ); diff --git a/packages/spotlight/src/telemetry/profile-depth.spec.ts b/packages/spotlight/src/telemetry/profile-depth.spec.ts index 73d406dda..ffa704382 100644 --- a/packages/spotlight/src/telemetry/profile-depth.spec.ts +++ b/packages/spotlight/src/telemetry/profile-depth.spec.ts @@ -1,5 +1,5 @@ import { expect, test } from "vitest"; -import { createTelemetryStore } from "../sdk/core.ts"; +import { createTelemetryStore, serializeTrace } from "../sdk/core.ts"; const depth = 12000; @@ -74,6 +74,21 @@ test("public ingestion retains and deterministically consolidates a 12000-frame current = current?.children?.[0]; } expect(current).toBeUndefined(); + const serialized = JSON.parse( + JSON.stringify(serializeTrace(trace)), + ) as ReturnType; + const spans = new Map(serialized.spans); + expect(spans.size).toBe(depth + 1); + for (const [id, span] of spans) { + for (const childId of span.childSpanIds) { + expect(spans.get(childId)?.parent_span_id).toBe(id); + } + if (id !== "root") { + expect( + spans.get(span.parent_span_id as string)?.childSpanIds, + ).toContain(id); + } + } const identities = [...trace.spans.keys()]; store.reset(); expect((await store.ingestEnvelope(input)).status).toBe("processed"); diff --git a/packages/spotlight/src/telemetry/profile-parent-links.spec.ts b/packages/spotlight/src/telemetry/profile-parent-links.spec.ts new file mode 100644 index 000000000..7b8522587 --- /dev/null +++ b/packages/spotlight/src/telemetry/profile-parent-links.spec.ts @@ -0,0 +1,134 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore, serializeTrace } from "../sdk/core.ts"; + +test.each([ + { + name: "appended thread children", + stacks: [["main"], ["other"], ["main"]], + rootOps: ["main", "other", "main"], + }, + { + name: "children adopted from an initially empty thread", + stacks: [[], ["main"], ["other"]], + rootOps: ["main", "other"], + }, + { + name: "adopted and appended frame children across sibling continuations", + stacks: [ + ["main"], + ["main", "left"], + ["main", "right"], + ["other"], + ["main", "last"], + ], + rootOps: ["main", "other", "main"], + }, + { + name: "consolidated children under retained thread wrappers", + stacks: [["main"], ["other"], ["main"], ["worker"]], + threads: ["main", "main", "main", "worker"], + rootOps: ["Thread", "Thread"], + }, + { + name: "adopted children under retained thread wrappers", + stacks: [[], ["main"], ["other"], ["worker"]], + threads: ["main", "main", "main", "worker"], + rootOps: ["Thread", "Thread"], + }, +])( + "public profile serialization preserves parent links for $name", + async ({ stacks, threads, rootOps }) => { + const store = createTelemetryStore(); + try { + const functions = [...new Set(stacks.flat())]; + expect( + ( + await store.ingestEnvelope([ + {}, + [ + [ + { type: "transaction" }, + { + event_id: "transaction", + type: "transaction", + transaction: "profile parent links", + start_timestamp: 1, + timestamp: 2, + contexts: { + trace: { + trace_id: "trace", + span_id: "root", + data: { "thread.id": "worker" }, + }, + }, + }, + ], + [ + { type: "profile" }, + { + event_id: "profile", + version: "1", + timestamp: 1, + platform: "javascript", + transaction: { trace_id: "trace", active_thread_id: "main" }, + profile: { + frames: functions.map((fn) => ({ + module: "app", + function: fn, + })), + // Sentry profiles encode stacks from leaf to root. + stacks: stacks.map((stack) => + stack.map((fn) => functions.indexOf(fn)).reverse(), + ), + samples: stacks.map((_, index) => ({ + thread_id: threads?.[index] ?? "main", + stack_id: index, + elapsed_since_start_ns: String((index + 1) * 100_000_000), + })), + }, + }, + ], + ], + ]) + ).status, + ).toBe("processed"); + + const trace = store.getTrace("trace")!; + // Exercise the actual JSON contract, where children are flat IDs. + const serialized = JSON.parse( + JSON.stringify(serializeTrace(trace)), + ) as ReturnType; + const spans = new Map(serialized.spans); + const root = spans.get("root")!; + expect(root.childSpanIds.map((id) => spans.get(id)?.op)).toEqual(rootOps); + expect(serialized.spanTree).toEqual(["root"]); + for (const [id, span] of spans) { + expect( + trace.spans.has(id), + `serialized span ${id} is in the public lookup`, + ).toBe(true); + for (const childId of span.childSpanIds) { + expect(spans.get(childId)?.parent_span_id, `child of ${id}`).toBe(id); + } + if (id !== "root") { + const parent = spans.get(span.parent_span_id as string); + expect(parent, `parent of ${id} resolves`).toBeDefined(); + expect(parent?.childSpanIds, `parent contains ${id}`).toContain(id); + } + } + if (stacks.length === 5) { + const firstMain = spans.get(root.childSpanIds[0])!; + expect(firstMain.childSpanIds.map((id) => spans.get(id)?.op)).toEqual([ + "left", + "right", + ]); + const lastMain = spans.get(root.childSpanIds[2])!; + expect(lastMain.childSpanIds.map((id) => spans.get(id)?.op)).toEqual([ + "last", + ]); + } + } finally { + store.dispose(); + } + }, +); From c5b96e5ff09850884e5b28dc74cb4a58e8ec90f7 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Thu, 29 Oct 2026 20:26:55 +0530 Subject: [PATCH 15/15] fix(spotlight): Keep root profile frames parentless Use a neutral graft parent for fillers between independent trace roots. Co-authored-by: Codex --- .../spotlight/src/telemetry/data/profiles.ts | 2 + .../telemetry/profile-parent-links.spec.ts | 85 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/packages/spotlight/src/telemetry/data/profiles.ts b/packages/spotlight/src/telemetry/data/profiles.ts index dc90e0553..858b9c281 100644 --- a/packages/spotlight/src/telemetry/data/profiles.ts +++ b/packages/spotlight/src/telemetry/data/profiles.ts @@ -295,6 +295,8 @@ export function graftProfileSpans( if (start < end) { graftProfileSpansIntoTree(trace, profile, trace.spanTree, { ...trace, + // The trace's span_id identifies a transaction, not a parent of its roots. + span_id: undefined, start_timestamp: start, timestamp: end, }); diff --git a/packages/spotlight/src/telemetry/profile-parent-links.spec.ts b/packages/spotlight/src/telemetry/profile-parent-links.spec.ts index 7b8522587..c1eeffa9d 100644 --- a/packages/spotlight/src/telemetry/profile-parent-links.spec.ts +++ b/packages/spotlight/src/telemetry/profile-parent-links.spec.ts @@ -1,6 +1,91 @@ import { expect, test } from "vitest"; import { createTelemetryStore, serializeTrace } from "../sdk/core.ts"; +test.each([false, true])( + "profile fillers between independent roots retain top-level parent links (multiple threads: %s)", + async (multipleThreads) => { + const store = createTelemetryStore(); + try { + const transaction = [ + { type: "transaction" }, + { + event_id: "transaction", + type: "transaction", + start_timestamp: 1, + timestamp: 1.4, + contexts: { + trace: { + trace_id: "trace", + span_id: "root", + data: { "thread.id": "main" }, + }, + profile: { profiler_id: "profiler" }, + }, + }, + ]; + const late = [ + { type: "span" }, + { + trace_id: "trace", + span_id: "late", + start_timestamp: 1.7, + timestamp: 1.8, + data: { "thread.id": "worker" }, + }, + ]; + const profile = [ + { type: "profile_chunk" }, + { + event_id: "chunk", + type: "profile_chunk", + version: "2", + profiler_id: "profiler", + chunk_id: "chunk", + platform: "javascript", + profile: { + frames: [{ module: "app", function: "gap" }], + stacks: [[0]], + samples: [1.5, 1.52, 1.56, 1.6].map((timestamp, index) => ({ + timestamp, + stack_id: 0, + thread_id: multipleThreads && index % 2 ? "worker" : "main", + })), + }, + }, + ]; + for (const items of [[transaction, profile], [late]]) { + await store.ingestEnvelope([ + { sdk: { name: "sentry.javascript.node", version: "10.63.0" } }, + items, + ]); + } + const trace = serializeTrace(store.getTrace("trace")!); + const spans = new Map(trace.spans); + const roots = new Set(trace.spanTree); + expect(trace.spanTree[0]).toBe("root"); + expect(trace.spanTree.at(-1)).toBe("late"); + expect(trace.spanTree.length).toBeGreaterThan(2); + expect(spans.get("root")?.childSpanIds).toEqual([]); + for (const [id, span] of spans) { + if (roots.has(id)) { + expect( + span.parent_span_id, + `top-level ${id} has no parent`, + ).toBeUndefined(); + } else { + const parent = spans.get(span.parent_span_id as string); + expect(parent?.childSpanIds, `parent contains ${id}`).toContain(id); + } + for (const childId of span.childSpanIds) { + expect(spans.get(childId)?.parent_span_id).toBe(id); + } + } + } finally { + store.dispose(); + } + }, +); + test.each([ { name: "appended thread children",