From 66d6a67c95d84a8e1f4821ca0f272a2e40be1287 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Tue, 27 Oct 2026 01:43:26 +0530 Subject: [PATCH 1/7] feat(sdk): Share portable telemetry processing across consumers --- packages/spotlight/README.md | 163 +++++- .../continuous_profiling/profile_chunk_1.txt | 2 +- packages/spotlight/examples/sdk/frontend.ts | 12 + packages/spotlight/examples/sdk/react.tsx | 16 + packages/spotlight/examples/sdk/server.ts | 44 ++ packages/spotlight/package.json | 18 +- packages/spotlight/src/sdk/core.ts | 22 + packages/spotlight/src/sdk/node.spec.ts | 318 ++++++++++++ packages/spotlight/src/sdk/node.ts | 163 ++++++ .../src/server/parser/processEnvelope.ts | 52 +- packages/spotlight/src/server/sdk.spec.ts | 414 +++++++++++++++ packages/spotlight/src/shared/constants.ts | 2 +- .../spotlight/src/telemetry/ai/aiLibraries.ts | 60 +++ packages/spotlight/src/telemetry/ai/genAI.ts | 357 +++++++++++++ .../spotlight/src/telemetry/ai/vercelAISDK.ts | 467 +++++++++++++++++ .../src/telemetry/attachments.spec.ts | 61 +++ .../spotlight/src/telemetry/client.spec.ts | 399 +++++++++++++++ .../src/telemetry/constants/performance.ts | 60 +++ .../src/telemetry/constants/sentry.ts | 14 + .../spotlight/src/telemetry/data/profiles.ts | 215 ++++++++ packages/spotlight/src/telemetry/index.ts | 340 ++++++++++++ .../src/telemetry/internal/engine.ts | 27 + .../src/telemetry/internal/envelopes.ts | 61 +++ .../src/telemetry/internal/readonly.ts | 59 +++ .../spotlight/src/telemetry/internal/types.ts | 3 + packages/spotlight/src/telemetry/logger.ts | 23 + .../spotlight/src/telemetry/parity.spec.ts | 88 ++++ packages/spotlight/src/telemetry/public.ts | 100 ++++ packages/spotlight/src/telemetry/queries.ts | 2 + .../spotlight/src/telemetry/selectors/ai.ts | 13 + .../src/telemetry/selectors/database.ts | 15 + .../src/telemetry/selectors/index.ts | 6 + .../src/telemetry/selectors/profiles.ts | 43 ++ .../src/telemetry/selectors/resources.ts | 15 + .../src/telemetry/selectors/selectors.test.ts | 96 ++++ .../src/telemetry/selectors/spans.ts | 39 ++ .../src/telemetry/selectors/webVitals.ts | 14 + .../spotlight/src/telemetry/store/helpers.ts | 6 + .../telemetry/store/slices/envelopesSlice.ts | 164 ++++++ .../src/telemetry/store/slices/eventsSlice.ts | 241 +++++++++ .../src/telemetry/store/slices/logsSlice.ts | 14 + .../telemetry/store/slices/profilesSlice.ts | 23 + .../src/telemetry/store/slices/sdksSlice.ts | 39 ++ .../src/telemetry/store/slices/sharedSlice.ts | 30 ++ .../store/slices/subscriptionsSlice.ts | 27 + .../src/telemetry/store/slices/tracesSlice.ts | 11 + .../spotlight/src/telemetry/store/types.ts | 107 ++++ .../spotlight/src/telemetry/store/utils.ts | 9 + .../src/telemetry/store/utils/logProcessor.ts | 45 ++ .../store/utils/profileChunkProcessor.ts | 157 ++++++ .../telemetry/store/utils/profileProcessor.ts | 75 +++ .../telemetry/store/utils/spanProcessor.ts | 106 ++++ .../telemetry/store/utils/traceInitializer.ts | 30 ++ .../telemetry/store/utils/traceProcessor.ts | 187 +++++++ packages/spotlight/src/telemetry/types.ts | 484 ++++++++++++++++++ .../src/telemetry/utils/profileUtils.ts | 15 + .../src/telemetry/utils/sdkToPlatform.ts | 15 + .../spotlight/src/telemetry/utils/sentry.ts | 42 ++ .../spotlight/src/telemetry/utils/traces.ts | 87 ++++ .../src/telemetry/utils/webVitals.ts | 167 ++++++ packages/spotlight/src/telemetry/uuid.ts | 9 + packages/spotlight/src/ui/lib/logger.ts | 24 +- packages/spotlight/src/ui/lib/uuid.ts | 10 +- .../components/events/EventContexts.test.tsx | 31 ++ .../components/events/EventContexts.tsx | 8 +- .../telemetry/components/insights/Queries.tsx | 53 +- .../components/insights/QuerySummary.tsx | 7 +- .../components/insights/Resources.tsx | 76 +-- .../insights/aiTraces/sdks/aiLibraries.ts | 61 +-- .../insights/aiTraces/sdks/genAI.ts | 358 +------------ .../insights/aiTraces/sdks/vercelAISDK.ts | 468 +---------------- .../insights/webVitals/WebVitalsDetail.tsx | 21 +- .../components/insights/webVitals/index.tsx | 28 +- .../spotlight/src/ui/telemetry/constants.ts | 63 +-- .../src/ui/telemetry/constants/sentry.ts | 14 +- .../src/ui/telemetry/data/profiles.ts | 207 +------- .../data/sentryEventsContext.test.tsx | 53 ++ .../ui/telemetry/data/sentryEventsContext.tsx | 58 +-- .../src/ui/telemetry/data/useSentrySpans.ts | 20 +- .../ui/telemetry/data/useSpotlightAITraces.ts | 67 +-- .../hooks/useErrorFiltering.test.tsx | 4 +- packages/spotlight/src/ui/telemetry/index.tsx | 2 +- .../ui/telemetry/store/contextLines.test.ts | 103 ++++ .../src/ui/telemetry/store/helpers.ts | 7 +- .../telemetry/store/slices/envelopesSlice.ts | 132 +---- .../ui/telemetry/store/slices/eventsSlice.ts | 214 +------- .../telemetry/store/slices/feedback.spec.ts | 4 +- .../ui/telemetry/store/slices/logsSlice.ts | 18 +- .../telemetry/store/slices/profilesSlice.ts | 60 +-- .../ui/telemetry/store/slices/sdksSlice.ts | 43 +- .../telemetry/store/slices/settingsSlice.ts | 21 - .../ui/telemetry/store/slices/sharedSlice.ts | 70 +-- .../ui/telemetry/store/slices/spans.spec.ts | 66 +-- .../store/slices/subscriptionsSlice.ts | 31 +- .../ui/telemetry/store/slices/tracesSlice.ts | 15 +- .../spotlight/src/ui/telemetry/store/store.ts | 169 +++++- .../spotlight/src/ui/telemetry/store/types.ts | 128 +---- .../spotlight/src/ui/telemetry/store/utils.ts | 10 +- .../ui/telemetry/store/utils/logProcessor.ts | 47 +- .../store/utils/profileChunkProcessor.ts | 158 +----- .../telemetry/store/utils/profileProcessor.ts | 76 +-- .../ui/telemetry/store/utils/spanProcessor.ts | 108 +--- .../telemetry/store/utils/traceInitializer.ts | 30 +- .../telemetry/store/utils/traceProcessor.ts | 164 +----- packages/spotlight/src/ui/telemetry/types.ts | 479 +---------------- packages/spotlight/src/ui/telemetry/utils.ts | 5 +- .../src/ui/telemetry/utils/profileUtils.ts | 16 +- .../src/ui/telemetry/utils/sdkToPlatform.ts | 16 +- .../src/ui/telemetry/utils/sentry.ts | 45 +- .../src/ui/telemetry/utils/traces.ts | 88 +--- .../src/ui/telemetry/utils/webVitals.ts | 168 +----- packages/spotlight/tests/types/sdk-core.ts | 29 ++ .../tests/types/tsconfig.sdk-core.json | 13 + packages/spotlight/vite.node.config.ts | 3 + pnpm-lock.yaml | 6 +- 115 files changed, 6391 insertions(+), 3677 deletions(-) create mode 100644 packages/spotlight/examples/sdk/frontend.ts create mode 100644 packages/spotlight/examples/sdk/react.tsx create mode 100644 packages/spotlight/examples/sdk/server.ts create mode 100644 packages/spotlight/src/sdk/core.ts create mode 100644 packages/spotlight/src/sdk/node.spec.ts create mode 100644 packages/spotlight/src/sdk/node.ts create mode 100644 packages/spotlight/src/server/sdk.spec.ts create mode 100644 packages/spotlight/src/telemetry/ai/aiLibraries.ts create mode 100644 packages/spotlight/src/telemetry/ai/genAI.ts create mode 100644 packages/spotlight/src/telemetry/ai/vercelAISDK.ts create mode 100644 packages/spotlight/src/telemetry/attachments.spec.ts create mode 100644 packages/spotlight/src/telemetry/client.spec.ts create mode 100644 packages/spotlight/src/telemetry/constants/performance.ts create mode 100644 packages/spotlight/src/telemetry/constants/sentry.ts create mode 100644 packages/spotlight/src/telemetry/data/profiles.ts create mode 100644 packages/spotlight/src/telemetry/index.ts create mode 100644 packages/spotlight/src/telemetry/internal/engine.ts create mode 100644 packages/spotlight/src/telemetry/internal/envelopes.ts create mode 100644 packages/spotlight/src/telemetry/internal/readonly.ts create mode 100644 packages/spotlight/src/telemetry/internal/types.ts create mode 100644 packages/spotlight/src/telemetry/logger.ts create mode 100644 packages/spotlight/src/telemetry/parity.spec.ts create mode 100644 packages/spotlight/src/telemetry/public.ts create mode 100644 packages/spotlight/src/telemetry/queries.ts create mode 100644 packages/spotlight/src/telemetry/selectors/ai.ts create mode 100644 packages/spotlight/src/telemetry/selectors/database.ts create mode 100644 packages/spotlight/src/telemetry/selectors/index.ts create mode 100644 packages/spotlight/src/telemetry/selectors/profiles.ts create mode 100644 packages/spotlight/src/telemetry/selectors/resources.ts create mode 100644 packages/spotlight/src/telemetry/selectors/selectors.test.ts create mode 100644 packages/spotlight/src/telemetry/selectors/spans.ts create mode 100644 packages/spotlight/src/telemetry/selectors/webVitals.ts create mode 100644 packages/spotlight/src/telemetry/store/helpers.ts create mode 100644 packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts create mode 100644 packages/spotlight/src/telemetry/store/slices/eventsSlice.ts create mode 100644 packages/spotlight/src/telemetry/store/slices/logsSlice.ts create mode 100644 packages/spotlight/src/telemetry/store/slices/profilesSlice.ts create mode 100644 packages/spotlight/src/telemetry/store/slices/sdksSlice.ts create mode 100644 packages/spotlight/src/telemetry/store/slices/sharedSlice.ts create mode 100644 packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts create mode 100644 packages/spotlight/src/telemetry/store/slices/tracesSlice.ts create mode 100644 packages/spotlight/src/telemetry/store/types.ts create mode 100644 packages/spotlight/src/telemetry/store/utils.ts create mode 100644 packages/spotlight/src/telemetry/store/utils/logProcessor.ts create mode 100644 packages/spotlight/src/telemetry/store/utils/profileChunkProcessor.ts create mode 100644 packages/spotlight/src/telemetry/store/utils/profileProcessor.ts create mode 100644 packages/spotlight/src/telemetry/store/utils/spanProcessor.ts create mode 100644 packages/spotlight/src/telemetry/store/utils/traceInitializer.ts create mode 100644 packages/spotlight/src/telemetry/store/utils/traceProcessor.ts create mode 100644 packages/spotlight/src/telemetry/types.ts create mode 100644 packages/spotlight/src/telemetry/utils/profileUtils.ts create mode 100644 packages/spotlight/src/telemetry/utils/sdkToPlatform.ts create mode 100644 packages/spotlight/src/telemetry/utils/sentry.ts create mode 100644 packages/spotlight/src/telemetry/utils/traces.ts create mode 100644 packages/spotlight/src/telemetry/utils/webVitals.ts create mode 100644 packages/spotlight/src/telemetry/uuid.ts create mode 100644 packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx create mode 100644 packages/spotlight/src/ui/telemetry/data/sentryEventsContext.test.tsx create mode 100644 packages/spotlight/src/ui/telemetry/store/contextLines.test.ts delete mode 100644 packages/spotlight/src/ui/telemetry/store/slices/settingsSlice.ts create mode 100644 packages/spotlight/tests/types/sdk-core.ts create mode 100644 packages/spotlight/tests/types/tsconfig.sdk-core.json diff --git a/packages/spotlight/README.md b/packages/spotlight/README.md index 8a1382527..33dc45800 100644 --- a/packages/spotlight/README.md +++ b/packages/spotlight/README.md @@ -1,5 +1,164 @@ # `@spotlightjs/spotlight` -This package combines the UI from `@spotlightjs/overlay` and the sidecar from `@spotlightjs/sidecar`. +Spotlight provides a development telemetry UI, a local sidecar, and a desktop app. +Its SDK shares the same normalization, trace assembly, and insight calculations +across server and frontend consumers. -This is the package to use for Spotlight. +## Choose an entry point + +| Entry | Factory | Input | +| --- | --- | --- | +| `@spotlightjs/spotlight/sdk/core` | `createTelemetry()` | Parsed Sentry envelopes; portable across server and frontend | +| `@spotlightjs/spotlight/sdk/node` | `createNodeTelemetry()` | Adds serialized bodies, decompression, and raw payload history | +| `@spotlightjs/spotlight/sdk` | Existing buffer helpers | Compatibility for existing transport consumers | + +The core requires no React, DOM, Node globals, or Zustand knowledge. Each instance +owns its state. The SDK does not start a server or synchronize instances; feed +server and frontend instances the same envelope stream when both need processing. +Match retention settings when comparing their retained histories. + +## Server: receive envelopes and read insights + +```ts +import { createNodeTelemetry } from "@spotlightjs/spotlight/sdk/node"; + +const telemetry = createNodeTelemetry({ + rawBufferSize: 100, + retention: { maxEnvelopes: 1000, maxAgeMs: 10 * 60 * 1000 }, +}); + +const result = await telemetry.ingest({ + body: Buffer.from(envelopeBody), + // Defaults to application/x-sentry-envelope. + // contentType: request.headers["content-type"], + // encoding: request.headers["content-encoding"], // identity, gzip, deflate, br + // userAgent: request.headers["user-agent"], +}); + +if (result.status === "invalid") { + console.error(result.diagnostics); +} else if (result.status === "processed") { + const trace = telemetry.getTrace(traceId); + const queries = telemetry.getQuerySummaries({ traceId }); + const raw = telemetry.getRawPayload(result.envelopeId); // defensive byte copy +} +``` + +See [the HTTP server example](examples/sdk/server.ts), which receives envelopes +at `POST /envelope` and serves query summaries at `GET /queries?traceId=...`. +Call `server.listen(...)` to start that example; closing it disposes its SDK instance. + +## Frontend: process the same stream + +```ts +import { createTelemetry } from "@spotlightjs/spotlight/sdk/core"; + +const telemetry = createTelemetry(); + +// Your SSE/WebSocket handler supplies parsed envelopes. +await telemetry.ingestEnvelope(parsedEnvelope); + +const queries = telemetry.getQuerySummaries({ traceId }); +// [{ description, count, totalDurationMs, averageDurationMs, spans }] +const logs = telemetry.getLogs({ traceId }); +const spans = telemetry.getSpans({ traceId }); +``` + +See [the frontend lifecycle example](examples/sdk/frontend.ts). Create an instance +once for the view or application, then call `dispose()` when it is no longer needed. + +## React: subscribe to completed snapshots + +```tsx +import { useSyncExternalStore } from "react"; +import type { Telemetry } from "@spotlightjs/spotlight/sdk/core"; + +function TraceCount({ telemetry }: { telemetry: Telemetry }) { + const snapshot = useSyncExternalStore( + telemetry.subscribe, + telemetry.getSnapshot, + telemetry.getSnapshot, + ); + return {snapshot.traces.length} traces; +} +``` + +See [the React query-list example](examples/sdk/react.tsx). Snapshots keep the same +identity until a committed update, making them suitable for `useSyncExternalStore`. +The core never exposes a writable store or requires a React adapter dependency. + +## Reads and units + +Collection reads accept filters; singular reads accept an identity. + +| Read | Result | +| --- | --- | +| `getTrace(traceId)`, `getTraces({ traceId? })` | Assembled traces | +| `getEvent(eventId)`, `getEvents({ traceId?, spanId? })` | Normalized events | +| `getErrors({ traceId?, spanId? })` | Captured exception events | +| `getSpans({ traceId? })` | Normalized spans | +| `getLog(logId)`, `getLogs({ traceId? })` | Logs and trace correlation | +| `getQuerySummaries({ traceId?, description? })` | Database spans grouped by trimmed description | +| `getResourceSummaries({ traceId? })` | Resource timings and encoded sizes | +| `getProfile(traceId)` | Associated V1 or continuous profile | +| `getProfileFunctionSummaries({ traceId? })` | Inclusive frame durations and sample counts | +| `getAITraces({ traceId? })` | AI interactions, tokens, prompts, responses, tools | +| `getWebVitals()` | Scored page-load events | +| `getEnvelope(envelopeId)`, `getEnvelopes()` | Retained wire envelopes | +| `getSdks()` | Observed SDK identities | + +Normalized event/span timestamps are Unix **milliseconds**. Summary fields name +units explicitly: `totalDurationMs`, `averageDurationMs`, `averageEncodedSizeBytes`. +Stored envelopes retain wire timestamp units. Parsed attachment strings and bytes +use the same `{ data }` representation as the Node decoder: UTF-8 for text content +types, base64 for binary data. Internal envelope metadata is added to the stored +copy. Caller input is never mutated. Telemetry reads return read-only snapshots, +arrays, objects, maps, and sets; copy data when you need to edit it. + +## Ingestion results and diagnostics + +Both factories return the same discriminated result from ingestion: + +- `processed`: accepted telemetry, with `envelopeId` and `diagnostics`. +- `ignored`: `reason` explains a duplicate, unsupported item/media type, reset, or + disposed instance. Valid unsupported-only envelopes are retained for inspection + and include `envelopeId`. +- `invalid`: malformed input or decompression failure, with `diagnostics`. + +Mixed span batches retain valid spans and report skipped malformed spans. Supply +`onDiagnostic(diagnostic)` to observe diagnostics, or inspect each ingestion result. +Optional `enrichStacktrace: async event => { ... }` adds source context without +requiring a context-lines server. Enrichment failure retains the captured error. + +Ingestion resolves after processing/enrichment. Each committed envelope produces +one coherent subscription update; duplicate or invalid input produces none unless +retention also removes expired data. Subscribers can read new raw Node payloads +at that same update. A subscriber or diagnostic observer cannot break ingestion. + +## Retention and lifecycle + +- Processed retention defaults to **500 envelopes**, including telemetry derived + from them. Set `retention.maxEnvelopes` to another positive integer. +- Optional `retention.maxAgeMs` measures age since receipt. Expiration runs while + idle, and `await telemetry.prune()` can enforce limits explicitly. +- Eviction rebuilds derived state from retained envelopes so traces, logs, profiles, + and insights stay consistent. Enrichment is reused; generated IDs remain stable. + Rebuilding costs CPU proportional to retained telemetry; use smaller limits for + large profiles or high-volume streams. +- Node raw history is independently bounded by `rawBufferSize` (default **500**; + **0** disables it). Raw bytes can outlive processed retention and are copied on read. +- `reset()` clears captured data and raw history, preserves subscriptions, and + invalidates in-flight/queued work. The instance remains usable. +- `dispose()` clears captured data, releases timers and subscriptions, invalidates + pending work, and closes the instance. Later ingestion returns `ignored/disposed`. +- Stable envelope/event/log/span IDs deduplicate delivery. Reusing an idless parsed + envelope object also deduplicates it. Distinct idless log envelopes are separate + observations, even when their content matches. + +## Compatibility and migration + +Released `createSpotlightBuffer`, `pushToSpotlightBuffer`, and `decompressBody` +exports remain available from `/sdk` with their existing behavior. Raw buffering +alone does not invoke processing. Use `/sdk/core` and `/sdk/node` for the new +processing API. Its factory and type names are independent of the package name, +so moving the package later only requires changing import paths. diff --git a/packages/spotlight/_fixtures/continuous_profiling/profile_chunk_1.txt b/packages/spotlight/_fixtures/continuous_profiling/profile_chunk_1.txt index d444752c9..65cb89f48 100644 --- a/packages/spotlight/_fixtures/continuous_profiling/profile_chunk_1.txt +++ b/packages/spotlight/_fixtures/continuous_profiling/profile_chunk_1.txt @@ -1,3 +1,3 @@ {"event_id":"3e11a5c9831f4e49939c0a81944ea2cb","sent_at":"2024-08-27T15:00:11.503779Z","sdk":{"name":"sentry.cocoa","version":"8.36.0"}} -{"type":"profile_chunk","length":1671} +{"type":"profile_chunk","length":1643} {"type":"profile_chunk","version":"2","profiler_id":"71bba98d90b545c39f2ae73f702d7ef4","chunk_id":"3e11a5c9831f4e49939c0a81944ea2cb","platform":"cocoa","release":"io.sentry.sample.iOS-Swift@8.36.0+1","environment":"simulator","client_sdk":{"name":"sentry.cocoa","version":"8.36.0"},"debug_meta":{"images":[{"debug_id":"5819FF25-01CB-3D32-B84F-0634B37D3BBC","image_addr":"0x00000001023a8000","type":"macho","image_size":16384,"code_file":"/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS17.2.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libLogRedirect.dylib"}]},"profile":{"samples":[{"timestamp":1724770811.503,"stack_id":0,"thread_id":"259"},{"timestamp":1724770811.513,"stack_id":1,"thread_id":"259"},{"timestamp":1724770811.523,"stack_id":2,"thread_id":"259"},{"timestamp":1724770811.533,"stack_id":3,"thread_id":"259"},{"timestamp":1724770811.543,"stack_id":2,"thread_id":"259"},{"timestamp":1724770811.553,"stack_id":1,"thread_id":"259"},{"timestamp":1724770811.563,"stack_id":0,"thread_id":"259"}],"stacks":[[0],[1,0],[2,1,0],[3,2,1,0]],"frames":[{"instruction_addr":"0x000000010232d144","function":"_main","filename":"main.m","lineno":14,"in_app":true},{"instruction_addr":"0x000000010232d200","function":"UIApplicationMain","module":"UIKit","in_app":false},{"instruction_addr":"0x000000010232d300","function":"-[AppDelegate application:didFinishLaunchingWithOptions:]","filename":"AppDelegate.swift","lineno":15,"in_app":true},{"instruction_addr":"0x000000010232d400","function":"-[AppDelegate setupSentry]","filename":"AppDelegate.swift","lineno":25,"in_app":true}],"thread_metadata":{"259":{"name":"main","priority":31}}}} diff --git a/packages/spotlight/examples/sdk/frontend.ts b/packages/spotlight/examples/sdk/frontend.ts new file mode 100644 index 000000000..cd4a85c03 --- /dev/null +++ b/packages/spotlight/examples/sdk/frontend.ts @@ -0,0 +1,12 @@ +import { createTelemetry } from "@spotlightjs/spotlight/sdk/core"; + +/** Keep one instance for the lifetime of the telemetry view. */ +export function createFrontendTelemetry() { + const telemetry = createTelemetry({ retention: { maxEnvelopes: 500 } }); + return { + telemetry, + // Pass the parsed envelopes received over SSE, WebSocket, or your own transport. + receiveEnvelope: (envelope: unknown) => telemetry.ingestEnvelope(envelope), + disconnect: telemetry.dispose, + }; +} diff --git a/packages/spotlight/examples/sdk/react.tsx b/packages/spotlight/examples/sdk/react.tsx new file mode 100644 index 000000000..828f3bf3e --- /dev/null +++ b/packages/spotlight/examples/sdk/react.tsx @@ -0,0 +1,16 @@ +import type { Telemetry } from "@spotlightjs/spotlight/sdk/core"; +import { useSyncExternalStore } from "react"; + +export function QueryList({ telemetry, traceId }: { telemetry: Telemetry; traceId?: string }) { + useSyncExternalStore(telemetry.subscribe, telemetry.getSnapshot, telemetry.getSnapshot); + const queries = telemetry.getQuerySummaries({ traceId }); + return ( + + ); +} diff --git a/packages/spotlight/examples/sdk/server.ts b/packages/spotlight/examples/sdk/server.ts new file mode 100644 index 000000000..cbec4563d --- /dev/null +++ b/packages/spotlight/examples/sdk/server.ts @@ -0,0 +1,44 @@ +import { createServer } from "node:http"; +import { createNodeTelemetry } from "@spotlightjs/spotlight/sdk/node"; + +/** Embed the SDK in an HTTP server; the SDK itself does not open a port. */ +export function createTelemetryServer() { + const telemetry = createNodeTelemetry({ + rawBufferSize: 100, + retention: { maxEnvelopes: 1000, maxAgeMs: 10 * 60 * 1000 }, + }); + const server = createServer(async (request, response) => { + try { + const url = new URL(request.url ?? "/", "http://localhost"); + response.setHeader("content-type", "application/json"); + if (request.method === "POST" && url.pathname === "/envelope") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const result = await telemetry.ingest({ + body: Buffer.concat(chunks), + contentType: request.headers["content-type"], + encoding: request.headers["content-encoding"], + userAgent: request.headers["user-agent"], + }); + response.statusCode = result.status === "invalid" ? 400 : 202; + response.end(JSON.stringify(result)); + return; + } + if (request.method === "GET" && url.pathname === "/queries") { + const traceId = url.searchParams.get("traceId") ?? undefined; + response.end(JSON.stringify(telemetry.getQuerySummaries({ traceId }))); + return; + } + response.statusCode = 404; + response.end(JSON.stringify({ error: "Not found" })); + } catch { + // An aborted request body rejects its async iterator. Keep the server usable. + if (!response.destroyed) { + response.statusCode = 400; + response.end(JSON.stringify({ error: "Could not read request" })); + } + } + }); + server.on("close", telemetry.dispose); + return { server, telemetry }; +} diff --git a/packages/spotlight/package.json b/packages/spotlight/package.json index ee7319b0f..30f5217ec 100644 --- a/packages/spotlight/package.json +++ b/packages/spotlight/package.json @@ -32,9 +32,7 @@ "test:e2e:electron": "playwright test tests/electron.test.ts", "sample": "node ./_fixtures/send_to_sidecar.cjs" }, - "files": [ - "dist" - ], + "files": ["dist"], "bin": { "spotlight": "./dist/run.js" }, @@ -49,6 +47,14 @@ "./sdk": { "import": "./dist/server/sdk.js", "types": "./dist/server/sdk.d.ts" + }, + "./sdk/core": { + "types": "./dist/sdk/core.d.ts", + "import": "./dist/sdk/core.js" + }, + "./sdk/node": { + "types": "./dist/sdk/node.d.ts", + "import": "./dist/sdk/node.js" } }, "dependencies": { @@ -69,7 +75,8 @@ "semver": "^7.7.3", "uuidv7": "^1.0.2", "yaml": "^2.8.3", - "zod": "^4" + "zod": "^4", + "zustand": "^5.0.3" }, "devDependencies": { "@fontsource/raleway": "catalog:", @@ -133,8 +140,7 @@ "vite-plugin-dts": "^4.5.4", "vite-plugin-electron": "^0.29.0", "vite-plugin-svgr": "^5.2.0", - "vitest": "catalog:", - "zustand": "^5.0.3" + "vitest": "catalog:" }, "volta": { "extends": "../../package.json" diff --git a/packages/spotlight/src/sdk/core.ts b/packages/spotlight/src/sdk/core.ts new file mode 100644 index 000000000..cd5e53a05 --- /dev/null +++ b/packages/spotlight/src/sdk/core.ts @@ -0,0 +1,22 @@ +export { createTelemetry } from "../telemetry/index.ts"; +export type { + Telemetry, + TelemetryOptions, + TelemetrySnapshot, + RawEnvelope, + ProcessedEvent, + ProcessedSpan, + ProcessedTrace, + ProcessedLog, + ProcessedProfile, + AITrace, + QuerySummary, + DatabaseQueryFilter, + ResourceSummary, + ProfileFunctionSummary, + EventFilter, + TraceFilter, + RetentionOptions, + Diagnostic, + IngestResult, +} from "../telemetry/public.ts"; diff --git a/packages/spotlight/src/sdk/node.spec.ts b/packages/spotlight/src/sdk/node.spec.ts new file mode 100644 index 000000000..e6009b882 --- /dev/null +++ b/packages/spotlight/src/sdk/node.spec.ts @@ -0,0 +1,318 @@ +// @vitest-environment node +import { readFileSync } from "node:fs"; +import { brotliCompressSync, deflateSync, gzipSync } from "node:zlib"; +import { describe, expect, test, vi } from "vitest"; +import { createNodeTelemetry } from "./node.ts"; + +const fixture = readFileSync(new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url)); + +describe("Node Telemetry adapter", () => { + test.each(["replay_recording", "future_binary"])( + "retains an opaque %s item alongside supported telemetry", + async type => { + const binary = deflateSync(Buffer.from("binary\nrecording")); + const body = Buffer.concat([ + Buffer.from('{}\n{"type":"event"}\n{"event_id":"with-opaque-item","timestamp":1}\n'), + Buffer.from(`${JSON.stringify({ type, length: binary.length })}\n`), + binary, + ]); + const telemetry = createNodeTelemetry(); + const result = await telemetry.ingest({ body }); + expect(result.status).toBe("processed"); + expect(telemetry.getEvent("with-opaque-item")).toBeDefined(); + expect(telemetry.getEnvelopes()[0][1][1][1]).toEqual({ data: binary.toString("base64") }); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + expect(telemetry.getRawPayload(result.envelopeId)?.body).toEqual(body); + telemetry.dispose(); + }, + ); + + test.each(['{}\n{"type":"attachment","length":0}', '{}\n{"type":"attachment"}'])( + "rejects a truncated item header without its terminating newline: %s", + async body => { + const telemetry = createNodeTelemetry(); + expect((await telemetry.ingest({ body })).status).toBe("invalid"); + expect(telemetry.getEnvelopes()).toEqual([]); + telemetry.dispose(); + }, + ); + + test("accepts an empty attachment payload after a complete item header", async () => { + const telemetry = createNodeTelemetry(); + expect((await telemetry.ingest({ body: '{}\n{"type":"attachment","length":0}\n' })).status).toBe("ignored"); + expect(telemetry.getEnvelopes()).toHaveLength(1); + telemetry.dispose(); + }); + + test.each([ + ["processed", fixture], + ["unsupported", Buffer.from('{}\n{"type":"attachment","length":5}\nhello')], + ] as const)("subscribers observe %s raw bytes together with the envelope in one notification", async (_, body) => { + const spotlight = createNodeTelemetry(); + const observations: (Buffer | undefined)[] = []; + spotlight.subscribe(snapshot => { + const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); + observations.push(spotlight.getRawPayload(id)?.body); + }); + await spotlight.ingest({ body }); + expect(observations).toEqual([body]); + }); + + test("raw history bounds already apply when subscribers observe the next envelope", async () => { + const spotlight = createNodeTelemetry({ rawBufferSize: 1 }); + const first = await spotlight.ingest({ body: fixture }); + if (first.status !== "processed") throw new Error("Expected processed envelope"); + const observations: { old: Buffer | undefined; newest: Buffer | undefined }[] = []; + spotlight.subscribe(snapshot => { + const envelope = snapshot.envelopes.find(item => String(item[0].__spotlight_envelope_id) !== first.envelopeId)!; + observations.push({ + old: spotlight.getRawPayload(first.envelopeId)?.body, + newest: spotlight.getRawPayload(String(envelope[0].__spotlight_envelope_id))?.body, + }); + }); + await spotlight.ingest({ body: fixture }); + expect(observations).toEqual([{ old: undefined, newest: fixture }]); + }); + + test("defaults to Sentry envelopes and exposes processed raw payloads", async () => { + const spotlight = createNodeTelemetry(); + const result = await spotlight.ingest({ body: fixture }); + expect(result.status).toBe("processed"); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(spotlight.getRawPayload(result.envelopeId)).toEqual({ + body: fixture, + contentType: "application/x-sentry-envelope", + }); + }); + + test.each([ + ["identity", (body: Buffer) => body], + ["gzip", gzipSync], + ["deflate", deflateSync], + ["br", brotliCompressSync], + ] as const)("decodes %s payloads", async (encoding, compress) => { + const spotlight = createNodeTelemetry(); + const result = await spotlight.ingest({ + body: new Uint8Array(compress(fixture)), + contentType: " Application/X-Sentry-Envelope ; charset=utf-8 ", + encoding, + }); + expect(result.status).toBe("processed"); + expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(fixture); + }); + + test("accepts serialized envelope strings", async () => { + const spotlight = createNodeTelemetry(); + expect((await spotlight.ingest({ body: fixture.toString("utf8") })).status).toBe("processed"); + }); + + test("returns an explicit ignored result for unsupported content types", async () => { + const spotlight = createNodeTelemetry(); + expect(await spotlight.ingest({ body: fixture, contentType: "application/json" })).toMatchObject({ + status: "ignored", + reason: expect.any(String), + }); + expect(spotlight.getEnvelopes()).toEqual([]); + }); + + test("retains raw payloads for valid envelopes containing only unsupported items", async () => { + const spotlight = createNodeTelemetry(); + const body = Buffer.from('{}\n{"type":"attachment","length":5}\nhello'); + const result = await spotlight.ingest({ body }); + expect(result).toMatchObject({ status: "ignored", reason: "unsupported_items", envelopeId: expect.any(String) }); + if (result.status !== "ignored" || !result.envelopeId) throw new Error("Expected retained unsupported envelope"); + expect(spotlight.getEnvelopes()).toHaveLength(1); + expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(body); + }); + + test.each([ + [{ body: fixture, encoding: "zstd" }, "unsupported_encoding"], + [{ body: fixture, encoding: "gzip" }, "decompression_failed"], + [{ body: "not an envelope" }, "invalid_envelope"], + [{ body: "null" }, "invalid_envelope"], + [{ body: "42" }, "invalid_envelope"], + [{ body: "[]" }, "invalid_envelope"], + [{ body: "{}\nnull\n{}" }, "invalid_envelope"], + [{ body: "{}\n42\n{}" }, "invalid_envelope"], + [{ body: "{}\n[]\n{}" }, "invalid_envelope"], + [{ body: '{}\n{"type":"event"}\nnot json' }, "invalid_envelope"], + [{ body: '{}\n{"type":"attachment","length":100}\nshort' }, "invalid_envelope"], + [{ body: '{}\n{"type":"attachment","length":-1}\nshort' }, "invalid_envelope"], + [{ body: '{}\n{"type":"attachment","length":1.5}\nshort' }, "invalid_envelope"], + [{ body: '{}\n{"type":"attachment","length":"5"}\nshort' }, "invalid_envelope"], + [{ body: '{}\n{"type":"attachment","length":3}\nshort' }, "invalid_envelope"], + [{ body: 42 }, "invalid_body"], + ])("returns diagnostics for malformed input %# without logging", async (input, code) => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const warnings = vi.spyOn(console, "warn").mockImplementation(() => {}); + const logs = vi.spyOn(console, "log").mockImplementation(() => {}); + try { + const spotlight = createNodeTelemetry(); + // Runtime validation is part of the public boundary. + const result = await spotlight.ingest(input as Parameters[0]); + expect(result).toMatchObject({ status: "invalid", diagnostics: [{ code }] }); + expect(spotlight.getEnvelopes()).toEqual([]); + expect(errors).not.toHaveBeenCalled(); + expect(warnings).not.toHaveBeenCalled(); + expect(logs).not.toHaveBeenCalled(); + } finally { + vi.restoreAllMocks(); + } + }); + + test("copies incoming and returned raw payload bytes", async () => { + const spotlight = createNodeTelemetry(); + const body = Buffer.from(fixture); + const pending = spotlight.ingest({ body }); + body.fill(0); + const result = await pending; + expect(result.status).toBe("processed"); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + const raw = spotlight.getRawPayload(result.envelopeId)!; + expect(raw.body).toEqual(fixture); + raw.body.fill(0); + expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(fixture); + }); + + test("bounds raw payload history independently from processed retention", async () => { + const spotlight = createNodeTelemetry({ rawBufferSize: 1 }); + const first = await spotlight.ingest({ body: fixture }); + const second = await spotlight.ingest({ body: fixture }); + if (first.status !== "processed" || second.status !== "processed") throw new Error("Expected envelopes"); + expect(spotlight.getRawPayload(first.envelopeId)).toBeUndefined(); + expect(spotlight.getRawPayload(second.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getEnvelopes()).toHaveLength(2); + spotlight.reset(); + expect(spotlight.getRawPayload(second.envelopeId)).toBeUndefined(); + expect(spotlight.getEnvelopes()).toEqual([]); + }); + + test("invalid ingestion cannot evict a retained raw payload", async () => { + const spotlight = createNodeTelemetry({ rawBufferSize: 1 }); + const first = await spotlight.ingest({ body: fixture }); + if (first.status !== "processed") throw new Error("Expected processed envelope"); + expect((await spotlight.ingest({ body: "not an envelope" })).status).toBe("invalid"); + expect((await spotlight.ingest({ body: "{}\n{}\n{}" })).status).toBe("invalid"); + expect(spotlight.getRawPayload(first.envelopeId)?.body).toEqual(fixture); + expect(spotlight).not.toHaveProperty("buffer"); + expect(spotlight).not.toHaveProperty("store"); + }); + + test("retains at most 500 raw payloads by default", async () => { + const spotlight = createNodeTelemetry({ retention: { maxEnvelopes: 1 } }); + const first = await spotlight.ingest({ body: fixture }); + let last = first; + for (let i = 0; i < 500; i++) last = await spotlight.ingest({ body: fixture }); + if (first.status !== "processed" || last.status !== "processed") throw new Error("Expected envelopes"); + expect(spotlight.getRawPayload(first.envelopeId)).toBeUndefined(); + expect(spotlight.getRawPayload(last.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getEnvelopes()).toHaveLength(1); + }); + + test("zero raw buffer capacity still processes telemetry", async () => { + const spotlight = createNodeTelemetry({ rawBufferSize: 0 }); + const result = await spotlight.ingest({ body: fixture }); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); + expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + }); + + test("processed age retention does not expire the separate raw history", async () => { + vi.useFakeTimers(); + const spotlight = createNodeTelemetry({ retention: { maxAgeMs: 100 } }); + try { + const result = await spotlight.ingest({ body: fixture }); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + await vi.advanceTimersByTimeAsync(101); + expect(spotlight.getEnvelopes()).toEqual([]); + expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(fixture); + } finally { + spotlight.dispose(); + vi.useRealTimers(); + } + }); + + test.each(["reset", "dispose"] as const)("%s during enrichment cannot restore raw history", async action => { + let start!: () => void; + let release!: () => void; + const started = new Promise(resolve => { + start = resolve; + }); + const gate = new Promise(resolve => { + release = resolve; + }); + const spotlight = createNodeTelemetry({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const pending = spotlight.ingest({ + body: [ + "{}", + '{"type":"event"}', + JSON.stringify({ + event_id: "pending-error", + timestamp: 1, + exception: { values: [{ type: "Error", value: "broken" }] }, + }), + ].join("\n"), + }); + await started; + spotlight[action](); + release(); + const result = await pending; + expect(result.status).toBe("ignored"); + if ("envelopeId" in result && result.envelopeId) expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); + expect(spotlight.getEnvelopes()).toEqual([]); + }); + + test("delivers invalid-input diagnostics to the configured observer", async () => { + const diagnostics: string[] = []; + const spotlight = createNodeTelemetry({ + onDiagnostic: diagnostic => { + diagnostics.push(diagnostic.code); + }, + }); + const result = await spotlight.ingest({ body: fixture, encoding: "zstd" }); + expect(result.status).toBe("invalid"); + expect(diagnostics).toEqual(["unsupported_encoding"]); + }); + + test("throwing diagnostic observers do not reject invalid input", async () => { + const spotlight = createNodeTelemetry({ + onDiagnostic: () => { + throw new Error("observer failed"); + }, + }); + expect((await spotlight.ingest({ body: fixture, encoding: "zstd" })).status).toBe("invalid"); + }); + + test("reset from a subscriber cannot repopulate raw payload history", async () => { + const spotlight = createNodeTelemetry(); + let reset = false; + spotlight.subscribe(() => { + if (!reset && spotlight.getEnvelopes().length > 0) { + reset = true; + spotlight.reset(); + } + }); + const result = await spotlight.ingest({ body: fixture }); + expect(reset).toBe(true); + if ("envelopeId" in result && result.envelopeId) expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); + expect(spotlight.getEnvelopes()).toEqual([]); + }); + + test("dispose clears raw payloads and prevents subsequent ingestion", async () => { + const spotlight = createNodeTelemetry(); + const result = await spotlight.ingest({ body: fixture }); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + spotlight.dispose(); + expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); + expect((await spotlight.ingest({ body: fixture })).status).toBe("ignored"); + expect(spotlight.getEnvelopes()).toEqual([]); + }); +}); diff --git a/packages/spotlight/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts new file mode 100644 index 000000000..7692ca2d3 --- /dev/null +++ b/packages/spotlight/src/sdk/node.ts @@ -0,0 +1,163 @@ +import { Buffer } from "node:buffer"; +import { brotliDecompressSync, gunzipSync, inflateSync } from "node:zlib"; +import { processEnvelope } from "../server/parser/processEnvelope.ts"; +import { type Diagnostic, type IngestResult, type Telemetry, type TelemetryOptions, createTelemetry } from "./core.ts"; + +export type NodeTelemetryOptions = TelemetryOptions & { + /** + * Maximum decoded raw payloads to retain (default 500); zero disables storage. + * This history is independent of processed telemetry retention and prune(). + */ + rawBufferSize?: number; +}; + +export type NodeIngestInput = { + body: Uint8Array | string; + contentType?: string; + encoding?: string; + userAgent?: string; +}; + +export type RawPayload = { body: Buffer; contentType: string }; +export type NodeTelemetry = Telemetry & { + ingest(input: NodeIngestInput): Promise; + getRawPayload(envelopeId: string): RawPayload | undefined; +}; + +const envelopeContentType = "application/x-sentry-envelope"; +const decompressors = new Map Buffer>([ + ["gzip", gunzipSync], + ["deflate", inflateSync], + ["br", brotliDecompressSync], +]); + +/** Decode serialized Sentry envelopes into an independent Telemetry processing instance. */ +export function createNodeTelemetry(options: NodeTelemetryOptions = {}): NodeTelemetry { + const rawBufferSize = options.rawBufferSize ?? 500; + if (!Number.isSafeInteger(rawBufferSize) || rawBufferSize < 0) { + throw new RangeError("rawBufferSize must be a non-negative safe integer"); + } + const telemetry = createTelemetry(options); + const rawPayloads = new Map(); + const stagedPayloads = new Map(); + let generation = 0; + let disposed = false; + + function retainRawPayload(envelopeId: string, payload: RawPayload) { + rawPayloads.set(envelopeId, payload); + while (rawPayloads.size > rawBufferSize) rawPayloads.delete(rawPayloads.keys().next().value!); + } + + // Register before public listeners so raw history commits with the core snapshot. + telemetry.subscribe(snapshot => { + for (const envelope of snapshot.envelopes) { + const envelopeId = String(envelope[0].__spotlight_envelope_id); + const payload = stagedPayloads.get(envelopeId); + if (!payload) continue; + stagedPayloads.delete(envelopeId); + retainRawPayload(envelopeId, payload); + } + }); + + function invalid(code: string, message: string): IngestResult { + const diagnostic: Diagnostic = { code, message }; + try { + options.onDiagnostic?.(diagnostic); + } catch { + // A diagnostic observer cannot turn invalid input into a rejected ingestion. + } + return { status: "invalid", diagnostics: [diagnostic] }; + } + + return { + ...telemetry, + async ingest(input) { + if (disposed) return { status: "ignored", reason: "disposed", diagnostics: [] }; + const currentGeneration = generation; + if (!input || (typeof input.body !== "string" && !(input.body instanceof Uint8Array))) { + return invalid("invalid_body", "Envelope body must be a string or Uint8Array"); + } + if (input.contentType !== undefined && typeof input.contentType !== "string") { + return invalid("invalid_content_type", "Content type must be a string"); + } + const contentType = (input.contentType ?? envelopeContentType).split(";")[0].trim().toLowerCase(); + if (contentType !== envelopeContentType) { + return { status: "ignored", reason: "unsupported_content_type", diagnostics: [] }; + } + if (input.encoding !== undefined && typeof input.encoding !== "string") { + return invalid("unsupported_encoding", "Content encoding must be a string"); + } + const encoding = input.encoding?.trim().toLowerCase(); + const decompress = encoding ? decompressors.get(encoding) : undefined; + if (encoding && encoding !== "identity" && !decompress) { + return invalid("unsupported_encoding", `Unsupported content encoding: ${encoding}`); + } + + // Copy before the first await so caller mutations cannot affect parsing or raw history. + let body: Buffer = typeof input.body === "string" ? Buffer.from(input.body, "utf8") : Buffer.from(input.body); + if (decompress) { + try { + body = decompress(body); + } catch { + return invalid("decompression_failed", "Could not decompress the envelope body"); + } + } + + let parseFailed = false; + let parsed: ReturnType = null; + try { + parsed = processEnvelope( + { contentType, data: body }, + input.userAgent, + () => { + parseFailed = true; + }, + true, + ); + } catch { + parseFailed = true; + } + if (!parsed || parseFailed) return invalid("invalid_envelope", "Malformed serialized Sentry envelope"); + + const envelopeId = String(parsed.envelope[0].__spotlight_envelope_id); + const payload = { body, contentType }; + if (rawBufferSize > 0) stagedPayloads.set(envelopeId, payload); + try { + const result = await telemetry.ingestEnvelope(parsed.envelope); + const retainPayload = + result.status === "processed" || (result.status === "ignored" && result.reason === "unsupported_items"); + if ( + retainPayload && + "envelopeId" in result && + result.envelopeId && + generation === currentGeneration && + !disposed && + stagedPayloads.get(envelopeId) === payload + ) { + // Age retention can remove an envelope before publication; raw history remains independent. + retainRawPayload(result.envelopeId, payload); + } + return result; + } finally { + if (stagedPayloads.get(envelopeId) === payload) stagedPayloads.delete(envelopeId); + } + }, + getRawPayload(envelopeId) { + const payload = rawPayloads.get(envelopeId); + return payload && { body: Buffer.from(payload.body), contentType: payload.contentType }; + }, + reset() { + generation += 1; + rawPayloads.clear(); + stagedPayloads.clear(); + telemetry.reset(); + }, + dispose() { + disposed = true; + generation += 1; + rawPayloads.clear(); + stagedPayloads.clear(); + telemetry.dispose(); + }, + }; +} diff --git a/packages/spotlight/src/server/parser/processEnvelope.ts b/packages/spotlight/src/server/parser/processEnvelope.ts index e0d77d574..268cc0b43 100644 --- a/packages/spotlight/src/server/parser/processEnvelope.ts +++ b/packages/spotlight/src/server/parser/processEnvelope.ts @@ -1,5 +1,6 @@ import type { Envelope, EnvelopeItem } from "@sentry/core"; import { type UUID, uuidv7obj } from "uuidv7"; +import { SUPPORTED_EVENT_TYPES } from "../../telemetry/constants/sentry.ts"; import { RAW_TYPES } from "../constants.ts"; import { type SourceType, inferEnvelopeSource } from "../formatters/human/utils.ts"; import { logger } from "../logger.ts"; @@ -37,9 +38,16 @@ const TEXT_CONTENT_TYPES = new Set([ * @see https://develop.sentry.dev/sdk/envelopes/#serialization-format * @param rawEvent Envelope data * @param senderUserAgent Optional User-Agent header from the HTTP request sending the envelope + * @param reportError Optional error observer; defaults to the legacy sidecar logger + * @param strict Reject truncated payloads and invalid explicit lengths * @returns parsed envelope */ -export function processEnvelope(rawEvent: RawEventContext, senderUserAgent?: string): ParsedEnvelope | null { +export function processEnvelope( + rawEvent: RawEventContext, + senderUserAgent?: string, + reportError: (error: unknown) => void = error => logger.error(error), + strict = false, +): ParsedEnvelope | null { let buffer = typeof rawEvent.data === "string" ? Uint8Array.from(rawEvent.data, c => c.charCodeAt(0)) : rawEvent.data; function readLine(length?: number) { @@ -49,9 +57,9 @@ export function processEnvelope(rawEvent: RawEventContext, senderUserAgent?: str return line; } - const envelopeHeader = parseJSONFromBuffer(readLine()) as Envelope[0]; - if (!envelopeHeader) { - logger.error("Malformed envelope header, skipping..."); + const envelopeHeader = parseJSONFromBuffer(readLine(), reportError) as Envelope[0]; + if (!envelopeHeader || typeof envelopeHeader !== "object" || Array.isArray(envelopeHeader)) { + reportError("Malformed envelope header, skipping..."); return null; } @@ -64,17 +72,35 @@ export function processEnvelope(rawEvent: RawEventContext, senderUserAgent?: str const items: EnvelopeItem[] = []; while (buffer.length) { - const itemHeader = parseJSONFromBuffer(readLine()) as EnvelopeItem[0]; - if (!itemHeader) { - logger.error("Malformed envelope item header, skipping envelope..."); + if (strict && buffer.indexOf(0xa) === -1) { + reportError("Truncated envelope item header, skipping envelope..."); + return null; + } + const itemHeader = parseJSONFromBuffer(readLine(), reportError) as EnvelopeItem[0]; + if (!itemHeader || typeof itemHeader !== "object" || Array.isArray(itemHeader)) { + reportError("Malformed envelope item header, skipping envelope..."); return null; } const payloadLength = itemHeader.length; + if ( + strict && + payloadLength !== undefined && + (!Number.isInteger(payloadLength) || + payloadLength < 0 || + payloadLength > buffer.length || + (payloadLength < buffer.length && buffer[payloadLength] !== 0xa)) + ) { + reportError("Malformed envelope item length, skipping envelope..."); + return null; + } const itemPayloadRaw = readLine(payloadLength); let itemPayload: EnvelopeItem[1]; try { - if (RAW_TYPES.has(itemHeader.type)) { + if ( + RAW_TYPES.has(itemHeader.type) || + (strict && !SUPPORTED_EVENT_TYPES.has(itemHeader.type) && itemHeader.type !== "span") + ) { const rawPayload = itemPayloadRaw as Buffer; if ( !itemHeader.content_type && @@ -88,9 +114,9 @@ export function processEnvelope(rawEvent: RawEventContext, senderUserAgent?: str data: rawPayload.toString(TEXT_CONTENT_TYPES.has(itemHeader.content_type as string) ? "utf-8" : "base64"), } as EnvelopeItem[1]; } else { - itemPayload = parseJSONFromBuffer(itemPayloadRaw); + itemPayload = parseJSONFromBuffer(itemPayloadRaw, reportError); if (!itemPayload) { - logger.error(itemHeader); + reportError(itemHeader); } else if (itemHeader.type) { // data sanitization // @ts-expect-error ts(2339) -- We should really stop adding type to payloads @@ -99,7 +125,7 @@ export function processEnvelope(rawEvent: RawEventContext, senderUserAgent?: str } } catch (err) { itemPayload = itemPayloadRaw; - logger.error(err); + reportError(err); } items.push([itemHeader, itemPayload] as EnvelopeItem); @@ -125,11 +151,11 @@ function getLineEnd(data: Uint8Array): number { return end; } -function parseJSONFromBuffer(data: Uint8Array): T { +function parseJSONFromBuffer(data: Uint8Array, reportError: (error: unknown) => void): T { try { return JSON.parse(new TextDecoder().decode(data)) as T; } catch (err) { - logger.error(err); + reportError(err); return null as T; } } diff --git a/packages/spotlight/src/server/sdk.spec.ts b/packages/spotlight/src/server/sdk.spec.ts new file mode 100644 index 000000000..0e534af52 --- /dev/null +++ b/packages/spotlight/src/server/sdk.spec.ts @@ -0,0 +1,414 @@ +import { readFileSync } from "node:fs"; +import { gzipSync } from "node:zlib"; +// @vitest-environment node +import type { Envelope } from "@sentry/core"; +import { describe, expect, test } from "vitest"; +import { createNodeTelemetry } from "../sdk/node.ts"; +import * as sdk from "./sdk.ts"; + +const traceId = "a9eab6940aba4ae7bf2f8ac5a54c6a2f"; +const rootId = "971f7be43c3ad277"; +const queryId = "832028f3635102b6"; +const fixture = readFileSync(new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url)); +const contentType = "application/x-sentry-envelope"; + +function envelope(type: string, payload: unknown, id = "test-envelope"): Envelope { + return [ + { sdk: { name: "sentry.javascript.node", version: "11.9.0" }, __spotlight_envelope_id: id }, + [[{ type }, payload]], + ] as unknown as Envelope; +} + +function span(overrides: Record = {}) { + return { + trace_id: traceId, + span_id: queryId, + parent_span_id: rootId, + name: " SELECT 1 ", + start_timestamp: 1793000000.125, + end_timestamp: 1793000000.25, + attributes: { "sentry.op": { type: "string", value: "db" } }, + ...overrides, + }; +} + +describe("shared Telemetry SDK", () => { + test("ingests compressed v2 spans into traces and query summaries without a browser", async () => { + const spotlight = createNodeTelemetry(); + const result = await spotlight.ingest({ body: gzipSync(fixture), encoding: "gzip", contentType }); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + expect(spotlight.getQuerySummaries()).toMatchObject([ + { + count: 1, + totalDurationMs: 125, + averageDurationMs: 125, + description: 'SELECT "public"."Customer"."id" FROM "public"."Customer"', + }, + ]); + expect(spotlight.getTraces()).toHaveLength(1); + expect(spotlight.getTrace(traceId)?.spanTree[0].children?.[0].span_id).toBe(queryId); + expect(spotlight.getSpans({ traceId })).toHaveLength(2); + expect(spotlight.getEnvelope(result.envelopeId)?.[1][0][1]).toMatchObject({ version: 2, items: [{}, {}] }); + expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getEvents()[0]).toMatchObject({ + timestamp: 1793000000250, + span: { timestamp: 1793000000250, start_timestamp: 1793000000125 }, + }); + }); + + test("does not modify parsed envelopes or multiply their timestamps when reused", async () => { + const input = envelope("transaction", { + type: "transaction", + event_id: "txn", + transaction: "request", + timestamp: 1793000001, + start_timestamp: 1793000000, + contexts: { trace: { trace_id: traceId, span_id: rootId } }, + spans: [ + { + trace_id: traceId, + span_id: queryId, + parent_span_id: rootId, + op: "db.query", + description: "SELECT 1", + start_timestamp: 1793000000.125, + timestamp: 1793000000.25, + }, + ], + }); + const before = JSON.stringify(input); + const first = createNodeTelemetry(); + const second = createNodeTelemetry(); + await first.ingestEnvelope(input); + await second.ingestEnvelope(input); + await first.ingestEnvelope(input); + expect(JSON.stringify(input)).toBe(before); + expect(first.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(second.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + }); + + test("keeps child spans when parents arrive later and deduplicates repeated delivery", async () => { + const spotlight = createNodeTelemetry(); + await spotlight.ingestEnvelope(envelope("span", { version: 2, items: [span()] })); + await spotlight.ingestEnvelope( + envelope( + "span", + { + version: 2, + items: [ + span({ span_id: rootId, parent_span_id: undefined, name: "request", is_segment: true, attributes: {} }), + ], + }, + "parent", + ), + ); + await spotlight.ingestEnvelope(envelope("span", { version: 2, items: [span()] }, "redelivery")); + expect(spotlight.getQuerySummaries()).toMatchObject([ + { count: 1, description: "SELECT 1", averageDurationMs: 125 }, + ]); + expect(spotlight.getTrace(traceId)?.spanTree[0].children?.map(s => s.span_id)).toEqual([queryId]); + expect(spotlight.getTrace(traceId)?.transactions).toHaveLength(1); + }); + + test("can scope queries to a trace without merging spans that share an ID", async () => { + const spotlight = createNodeTelemetry(); + await spotlight.ingestEnvelope( + envelope("span", { version: 2, items: [span(), span({ trace_id: "other-trace" })] }), + ); + expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 2, totalDurationMs: 250 }]); + expect(spotlight.getQuerySummaries({ traceId })).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(spotlight.getSpans({ traceId: "absent" })).toEqual([]); + }); + + test.each([true, false])( + "merges a legacy transaction and streamed segment for the same span (legacy first: %s)", + async legacyFirst => { + const spotlight = createNodeTelemetry(); + const legacy = envelope( + "transaction", + { + event_id: "legacy", + transaction: "request", + start_timestamp: 1793000000, + timestamp: 1793000001, + contexts: { trace: { trace_id: traceId, span_id: rootId, op: "http.server", data: { legacy: true } } }, + spans: [ + { + trace_id: traceId, + span_id: "legacy-query", + parent_span_id: rootId, + op: "db.query", + description: "SELECT 1", + start_timestamp: 1793000000.5, + timestamp: 1793000000.525, + }, + ], + }, + "legacy", + ); + const streamed = envelope( + "span", + { + version: 2, + items: [ + span({ name: "SELECT 1" }), + span({ + span_id: rootId, + parent_span_id: undefined, + is_segment: true, + name: "request", + start_timestamp: 1793000000, + end_timestamp: 1793000001, + attributes: { streamed: { value: "yes" } }, + }), + ], + }, + "streamed", + ); + for (const input of legacyFirst ? [legacy, streamed] : [streamed, legacy]) await spotlight.ingestEnvelope(input); + const trace = spotlight.getTrace(traceId)!; + expect(trace.transactions).toHaveLength(1); + expect(trace.rootTransactionName).toBe("request"); + expect(trace.spans.get(rootId)?.op).toBe("http.server"); + expect(trace.spans.get(rootId)?.data).toMatchObject({ legacy: true, streamed: "yes" }); + expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 2, totalDurationMs: 150 }]); + }, + ); + + test("isolates instance state and resets buffer and processed data together", async () => { + const first = createNodeTelemetry(); + const second = createNodeTelemetry(); + const ingested = await first.ingest({ body: fixture, contentType }); + expect(second.getTraces()).toEqual([]); + expect(second.getSdks()).toEqual([]); + first.reset(); + expect(first.getTraces()).toEqual([]); + expect(first.getQuerySummaries()).toEqual([]); + expect(first.getEnvelopes()).toEqual([]); + if (ingested.status === "processed") expect(first.getRawPayload(ingested.envelopeId)).toBeUndefined(); + }); + + test("awaits optional stacktrace enrichment before exposing errors and trace correlation", async () => { + const spotlight = createNodeTelemetry({ + enrichStacktrace: async event => { + await Promise.resolve(); + event.exception!.values![0].stacktrace!.frames[0].context_line = "throw new Error()"; + }, + }); + const input = envelope("event", { + event_id: "error", + type: "event", + timestamp: 1793000000.5, + exception: { values: [{ type: "Error", value: "broken", stacktrace: { frames: [{ filename: "app.ts" }] } }] }, + contexts: { trace: { trace_id: traceId, span_id: rootId } }, + }); + await spotlight.ingestEnvelope(input); + expect(spotlight.getErrors()).toHaveLength(1); + expect(spotlight.getErrors()[0].exception?.values?.[0].stacktrace?.frames[0].context_line).toBe( + "throw new Error()", + ); + expect(spotlight.getTrace(traceId)?.errors).toBe(1); + expect(JSON.stringify(input)).not.toContain("throw new Error()"); + }); + + test("reset prevents a pending error enrichment from restoring cleared data", async () => { + let release!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + const spotlight = createNodeTelemetry({ enrichStacktrace: () => gate }); + const pending = spotlight.ingestEnvelope( + envelope("event", { + event_id: "error", + type: "event", + timestamp: 1, + exception: { values: [{ type: "Error", value: "old" }] }, + contexts: { trace: { trace_id: traceId, span_id: rootId } }, + }), + ); + spotlight.reset(); + release(); + await pending; + expect(spotlight.getEvents()).toEqual([]); + expect(spotlight.getTraces()).toEqual([]); + }); + + test.each(["sdks", "logs", "events"] as const)( + "reset from a %s subscriber cancels the current ingestion", + async key => { + const spotlight = createNodeTelemetry(); + let reset = false; + spotlight.subscribe(state => { + if (!reset && state[key].length > 0) { + reset = true; + spotlight.reset(); + } + }); + const input = + key === "logs" + ? envelope("log", { items: [{ body: "hello", timestamp: 1, attributes: {} }] }) + : envelope("transaction", { + event_id: "transaction", + timestamp: 2, + start_timestamp: 1, + contexts: { trace: { trace_id: traceId, span_id: rootId } }, + }); + await spotlight.ingestEnvelope(input); + expect(reset).toBe(true); + expect(spotlight.getEvents()).toEqual([]); + expect(spotlight.getTraces()).toEqual([]); + expect(spotlight.getLogs()).toEqual([]); + expect(spotlight.getSdks()).toEqual([]); + expect(spotlight.getEnvelopes()).toEqual([]); + }, + ); + + test("normalizes logs and correlates them to traces", async () => { + const spotlight = createNodeTelemetry(); + await spotlight.ingestEnvelope( + envelope("log", { + type: "log", + items: [{ timestamp: 1793000000, body: "hello", level: "info", trace_id: traceId, attributes: {} }], + }), + ); + expect(spotlight.getLogs({ traceId })).toMatchObject([{ body: "hello", timestamp: 1793000000000 }]); + }); + + test("redelivery of the same idless parsed envelope does not duplicate logs", async () => { + const spotlight = createNodeTelemetry(); + const input = [ + {}, + [[{ type: "log" }, { items: [{ timestamp: 1, body: "hello", level: "info", attributes: {} }] }]], + ] as unknown as Envelope; + const before = JSON.stringify(input); + await spotlight.ingestEnvelope(input); + await spotlight.ingestEnvelope(input); + expect(spotlight.getLogs()).toHaveLength(1); + expect(spotlight.getEnvelopes()).toHaveLength(1); + expect(JSON.stringify(input)).toBe(before); + spotlight.reset(); + await spotlight.ingestEnvelope(input); + expect(spotlight.getLogs()).toHaveLength(1); + }); + + test("notifies subscribers with completed state and supports unsubscribing", async () => { + const spotlight = createNodeTelemetry(); + const observed: number[] = []; + const unsubscribe = spotlight.subscribe(() => observed.push(spotlight.getQuerySummaries().length)); + await spotlight.ingest({ body: fixture, contentType }); + expect(observed.at(-1)).toBe(1); + unsubscribe(); + const count = observed.length; + spotlight.reset(); + expect(observed).toHaveLength(count); + }); + + test("keeps the existing raw buffer SDK usable independently", () => { + const buffer = sdk.createSpotlightBuffer(10); + const container = sdk.pushToSpotlightBuffer({ spotlightBuffer: buffer, body: fixture, contentType }); + expect(container?.getEventTypes()).toEqual(["span"]); + }); + + test("concurrent ingestion of the same envelope waits for its pending processing", async () => { + let release!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + const spotlight = createNodeTelemetry({ enrichStacktrace: () => gate }); + const input = envelope("event", { event_id: "pending", type: "event", timestamp: 1, exception: { values: [] } }); + const first = spotlight.ingestEnvelope(input); + let secondFinished = false; + const second = spotlight.ingestEnvelope(input).then(() => { + secondFinished = true; + }); + await new Promise(resolve => setImmediate(resolve)); + expect(secondFinished).toBe(false); + release(); + await Promise.all([first, second]); + expect(spotlight.getErrors()).toHaveLength(1); + }); + + test("an unavailable enrichment adapter does not discard a captured error", async () => { + const spotlight = createNodeTelemetry({ + enrichStacktrace: async () => { + throw new Error("offline"); + }, + }); + await spotlight.ingestEnvelope( + envelope("event", { event_id: "error", type: "event", timestamp: 1, exception: { values: [] } }), + ); + expect(spotlight.getErrors()).toHaveLength(1); + }); + + test("shares resource summaries, AI extraction, and web-vitals scoring", async () => { + const spotlight = createNodeTelemetry(); + await spotlight.ingestEnvelope( + envelope("span", { + version: 2, + items: [ + span({ + span_id: "resource", + name: "app.js", + attributes: { "sentry.op": { value: "resource.script" }, "http.response_content_length": { value: 2048 } }, + }), + span({ + span_id: "ai", + name: "chat", + attributes: { + "sentry.op": { value: "gen_ai.chat" }, + "gen_ai.request.model": { value: "test-model" }, + "gen_ai.usage.input_tokens": { value: 10 }, + "gen_ai.usage.output_tokens": { value: 5 }, + }, + }), + ], + }), + ); + expect(spotlight.getResourceSummaries()).toMatchObject([ + { description: "app.js", count: 1, totalDurationMs: 125, averageEncodedSizeBytes: 2048 }, + ]); + expect(spotlight.getAITraces()).toMatchObject([ + { id: "ai", durationMs: 125, promptTokens: 10, completionTokens: 5 }, + ]); + await spotlight.ingestEnvelope( + envelope( + "transaction", + { + type: "transaction", + event_id: "page", + transaction: "/", + start_timestamp: 1, + timestamp: 2, + contexts: { trace: { trace_id: "page-trace", span_id: "page-span", op: "pageload" } }, + measurements: { fcp: { value: 900, unit: "millisecond" } }, + }, + "page", + ), + ); + const result = spotlight.getWebVitals(); + expect(result).toHaveLength(1); + expect(result[0].measurements["score.fcp"].value).toBeCloseTo(0.9, 5); + expect(result[0].measurements["score.total"].value).toBeCloseTo(0.192857, 5); + expect(spotlight.getEvent("page")?.measurements).not.toHaveProperty("score.total"); + }); + + test.each([true, false])( + "associates continuous profiles with their trace (profile first: %s)", + async profileFirst => { + const spotlight = createNodeTelemetry(); + const transaction = readFileSync( + new URL("../../_fixtures/continuous_profiling/transaction.txt", import.meta.url), + ); + const profile = readFileSync( + new URL("../../_fixtures/continuous_profiling/profile_chunk_1.txt", import.meta.url), + ); + for (const body of profileFirst ? [profile, transaction] : [transaction, profile]) { + await spotlight.ingest({ body, contentType }); + } + const profileTraceId = "f1e2d3c4b5a6978012345678901234ab"; + expect(spotlight.getProfile(profileTraceId)?.samples.length).toBeGreaterThan(0); + expect(spotlight.getTrace(profileTraceId)?.profileGrafted).toBe(true); + expect(spotlight.getProfileFunctionSummaries().length).toBeGreaterThan(0); + }, + ); +}); diff --git a/packages/spotlight/src/shared/constants.ts b/packages/spotlight/src/shared/constants.ts index c86a6f2c9..d1ebbca08 100644 --- a/packages/spotlight/src/shared/constants.ts +++ b/packages/spotlight/src/shared/constants.ts @@ -1,4 +1,4 @@ // Shared constants used by both UI and server export const CONTEXT_LINES_ENDPOINT = "/contextlines"; -export const RAW_TYPES = new Set(["attachment", "replay_video", "statsd"]); +export { RAW_TYPES } from "../telemetry/constants/sentry.ts"; export const SENTRY_CONTENT_TYPE = "application/x-sentry-envelope"; diff --git a/packages/spotlight/src/telemetry/ai/aiLibraries.ts b/packages/spotlight/src/telemetry/ai/aiLibraries.ts new file mode 100644 index 000000000..13960b69f --- /dev/null +++ b/packages/spotlight/src/telemetry/ai/aiLibraries.ts @@ -0,0 +1,60 @@ +import type { AILibraryHandler, AITrace, Span, Trace } from "../types.ts"; +import { genAIHandler } from "./genAI.ts"; +import { vercelAISDKHandler } from "./vercelAISDK.ts"; + +// Registry of AI span handlers. The generic gen_ai handler runs first because +// it only claims standard `gen_ai.*` spans that lack `vercel.ai.*` fields, so +// the Vercel handler still picks up its own proprietary spans. +const aiLibraries: AILibraryHandler[] = [genAIHandler, vercelAISDKHandler]; + +export { aiLibraries }; + +export function detectAILibraryHandler(span: Span): AILibraryHandler | null { + for (const handler of aiLibraries) { + // careful if we support multiple libraries, we should improve this method so it doesn't return the first one + if (handler.canHandleSpan(span)) { + return handler; + } + } + return null; +} + +export function extractAllAIRootSpans(spans: Span[]): { span: Span; handler: AILibraryHandler }[] { + const results: { span: Span; handler: AILibraryHandler }[] = []; + // Handlers can match overlapping span shapes (both AI handlers key off the + // `gen_ai.` op prefix). Attribute each root span to the first handler that + // claims it so a span is never surfaced twice. + const claimed = new Set(); + + for (const handler of aiLibraries) { + const rootSpans = handler.extractRootSpans(spans); + for (const rootSpan of rootSpans) { + if (claimed.has(rootSpan.span_id)) { + continue; + } + claimed.add(rootSpan.span_id); + results.push({ span: rootSpan, handler }); + } + } + + return results; +} + +export function createAITraceFromSpan(span: Span): AITrace | null { + const handler = detectAILibraryHandler(span); + if (!handler) { + return null; + } + + return handler.processTrace(span); +} + +export function hasAISpans(trace: Trace): boolean { + if (!trace.spanTree) { + return false; + } + + // TODO: We may want to cache/optimize this `extractAllAIRootSpans` helper + const aiRootSpans = extractAllAIRootSpans(trace.spanTree); + return aiRootSpans.length > 0; +} diff --git a/packages/spotlight/src/telemetry/ai/genAI.ts b/packages/spotlight/src/telemetry/ai/genAI.ts new file mode 100644 index 000000000..d028a91e3 --- /dev/null +++ b/packages/spotlight/src/telemetry/ai/genAI.ts @@ -0,0 +1,357 @@ +import { getAllSpansInTree } from "../store/helpers.ts"; +import type { AILibraryHandler, AIToolCall, AITrace, Span } from "../types.ts"; + +// Generic handler for the standardized `gen_ai.*` span conventions. Any SDK +// that follows sentry-conventions (sentry-python's PydanticAI/OpenAI/Anthropic +// integrations, the OTel GenAI conventions, ...) maps through here regardless +// of provider — the provider is just the `gen_ai.system` attribute, not a +// reason to fork the handler. The Vercel AI SDK keeps its own handler only +// because it emits proprietary `vercel.ai.*` fields on top of `gen_ai.*`. +// https://github.com/getsentry/sentry-conventions/tree/main/model/attributes/gen_ai +const GEN_AI_OP_PREFIX = "gen_ai."; +const VERCEL_FIELD_PREFIX = "vercel.ai."; + +// Span ops +const GEN_AI_CHAT_OP = "gen_ai.chat"; +const GEN_AI_INVOKE_AGENT_OP = "gen_ai.invoke_agent"; +const GEN_AI_EXECUTE_TOOL_OP = "gen_ai.execute_tool"; + +// Span data fields (sentry-conventions gen_ai.*) +const GEN_AI_OPERATION_NAME_FIELD = "gen_ai.operation.name"; +const GEN_AI_AGENT_NAME_FIELD = "gen_ai.agent.name"; +const GEN_AI_SYSTEM_FIELD = "gen_ai.system"; +const GEN_AI_REQUEST_MODEL_FIELD = "gen_ai.request.model"; +const GEN_AI_RESPONSE_MODEL_FIELD = "gen_ai.response.model"; +const GEN_AI_USAGE_INPUT_TOKENS_FIELD = "gen_ai.usage.input_tokens"; +const GEN_AI_USAGE_OUTPUT_TOKENS_FIELD = "gen_ai.usage.output_tokens"; +const GEN_AI_REQUEST_MESSAGES_FIELD = "gen_ai.request.messages"; +const GEN_AI_SYSTEM_INSTRUCTIONS_FIELD = "gen_ai.system_instructions"; +const GEN_AI_RESPONSE_TEXT_FIELD = "gen_ai.response.text"; +const GEN_AI_RESPONSE_TOOL_CALLS_FIELD = "gen_ai.response.tool_calls"; +const GEN_AI_RESPONSE_FINISH_REASONS_FIELD = "gen_ai.response.finish_reasons"; +const GEN_AI_TOOL_NAME_FIELD = "gen_ai.tool.name"; +const GEN_AI_TOOL_CALL_ID_FIELD = "gen_ai.tool.call.id"; +const GEN_AI_TOOL_INPUT_FIELD = "gen_ai.tool.input"; +const GEN_AI_TOOL_OUTPUT_FIELD = "gen_ai.tool.output"; + +const DEFAULT_TRACE_NAME = "AI Interaction"; +const UNKNOWN_OPERATION = "N/A"; + +const OPERATION_BADGE_MAP = new Map([ + ["chat", "Chat"], + ["invoke_agent", "Invoke Agent"], + ["execute_tool", "Execute Tool"], + [GEN_AI_CHAT_OP, "Chat"], + [GEN_AI_INVOKE_AGENT_OP, "Invoke Agent"], + [GEN_AI_EXECUTE_TOOL_OP, "Execute Tool"], +]); + +function isGenAISpan(span: Span): boolean { + return !!span.op?.toLowerCase().startsWith(GEN_AI_OP_PREFIX); +} + +/** + * Claims any standard `gen_ai.*` span that isn't a Vercel AI SDK span. Both + * carry the `gen_ai.` op prefix, but only the Vercel SDK sets `vercel.ai.*` + * data fields, so we defer those to the Vercel handler. A generic gen_ai span + * has at least one `gen_ai.*` data key and no `vercel.ai.*` key. + */ +function isGenAIConventionSpan(span: Span): boolean { + if (!isGenAISpan(span) || !span.data) { + return false; + } + + let hasGenAIField = false; + for (const key of Object.keys(span.data)) { + if (key.startsWith(VERCEL_FIELD_PREFIX)) { + return false; + } + if (key.startsWith(GEN_AI_OP_PREFIX)) { + hasGenAIField = true; + } + } + + return hasGenAIField; +} + +export const genAIHandler: AILibraryHandler = { + id: "gen-ai", + name: "Gen AI", + + canHandleSpan: isGenAIConventionSpan, + + extractRootSpans: (spans: Span[]): Span[] => { + const resultRoots: Span[] = []; + + const findAndCollectAIRoots = (spansToSearch: Span[]) => { + for (const currentSpan of spansToSearch) { + if (isGenAIConventionSpan(currentSpan)) { + resultRoots.push(currentSpan); + } else if (!isGenAISpan(currentSpan) && currentSpan.children?.length) { + // Stop at any gen_ai.* span: it's the shallowest AI root for its + // tree (claimed by this handler or the Vercel one), so its nested + // gen_ai.* children must not be surfaced as separate roots. + findAndCollectAIRoots(currentSpan.children); + } + } + }; + + findAndCollectAIRoots(spans); + return resultRoots; + }, + + processTrace: (rootSpan: Span): AITrace => { + const allSpans = getAllSpansInTree(rootSpan); + const tokenUsage = extractTokenUsage(allSpans); + const operation = determineOperation(rootSpan); + const hasToolCall = allSpans.some( + span => span.op === GEN_AI_EXECUTE_TOOL_OP || !!span.data?.[GEN_AI_TOOL_NAME_FIELD], + ); + + const durationMs = rootSpan.timestamp - rootSpan.start_timestamp; + + const trace: AITrace = { + id: rootSpan.span_id, + name: determineTraceName(rootSpan), + operation, + timestamp: rootSpan.start_timestamp, + durationMs, + tokensDisplay: formatTokensDisplay(tokenUsage.promptTokens, tokenUsage.completionTokens), + promptTokens: tokenUsage.promptTokens, + completionTokens: tokenUsage.completionTokens, + hasToolCall, + rawSpan: rootSpan, + metadata: { + metadata: {}, + promptTokens: tokenUsage.promptTokens, + completionTokens: tokenUsage.completionTokens, + }, + toolCalls: [], + }; + + const responseState = { finishReasonFromText: false }; + for (const span of allSpans) { + if (!span.data) continue; + extractMetadata(span, trace); + extractPromptData(span, trace); + extractResponseData(span, trace, responseState); + extractToolCallData(span, trace); + } + + return trace; + }, + + getDisplayTitle: (trace: AITrace): string => { + return trace.name || trace.operation || `AI Trace ${trace.id.substring(0, 8)}`; + }, + + getTypeBadge: (trace: AITrace): string => { + if (trace.hasToolCall) { + return "Tool-Call"; + } + + return OPERATION_BADGE_MAP.get(trace.operation) ?? trace.operation.replace(/^gen_ai\./, ""); + }, + + getTokensDisplay: (trace: AITrace): string => { + return trace.tokensDisplay; + }, +}; + +function determineOperation(rootSpan: Span): string { + const operationName = rootSpan.data?.[GEN_AI_OPERATION_NAME_FIELD]; + if (operationName) { + return String(operationName); + } + if (rootSpan.op) { + return rootSpan.op; + } + return UNKNOWN_OPERATION; +} + +function determineTraceName(rootSpan: Span): string { + const agentName = rootSpan.data?.[GEN_AI_AGENT_NAME_FIELD]; + if (agentName) { + return String(agentName); + } + return rootSpan.description || rootSpan.op || DEFAULT_TRACE_NAME; +} + +/** + * Sums token usage across every span in the tree so an agent root with multiple + * chat turns reports the total, not just the first turn's usage. + */ +function extractTokenUsage(spans: Span[]): { promptTokens?: number; completionTokens?: number } { + let promptTokens: number | undefined; + let completionTokens: number | undefined; + + for (const span of spans) { + if (!span.data) continue; + + if (span.data[GEN_AI_USAGE_INPUT_TOKENS_FIELD] !== undefined) { + promptTokens = (promptTokens ?? 0) + Number(span.data[GEN_AI_USAGE_INPUT_TOKENS_FIELD]); + } + + if (span.data[GEN_AI_USAGE_OUTPUT_TOKENS_FIELD] !== undefined) { + completionTokens = (completionTokens ?? 0) + Number(span.data[GEN_AI_USAGE_OUTPUT_TOKENS_FIELD]); + } + } + + return { promptTokens, completionTokens }; +} + +function formatTokensDisplay(promptTokens?: number, completionTokens?: number): string { + if (promptTokens !== undefined && completionTokens !== undefined) { + return `${promptTokens} / ${completionTokens}`; + } + if (promptTokens !== undefined) { + return `${promptTokens} / ?`; + } + if (completionTokens !== undefined) { + return `? / ${completionTokens}`; + } + return UNKNOWN_OPERATION; +} + +/** + * Normalizes message content from a string or an array of content blocks to a + * plain string. Only text blocks are surfaced; images/files would need UI work. + */ +function normalizeMessageContent(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .filter( + (block): block is { type: string; text: string } => block?.type === "text" && typeof block.text === "string", + ) + .map(block => block.text) + .join(""); + } + return ""; +} + +function extractMetadata(span: Span, trace: AITrace) { + if (!span.data) return; + + if (trace.metadata.modelId === undefined) { + const modelId = span.data[GEN_AI_REQUEST_MODEL_FIELD] ?? span.data[GEN_AI_RESPONSE_MODEL_FIELD]; + if (modelId !== undefined) { + trace.metadata.modelId = String(modelId); + } + } + + if (trace.metadata.modelProvider === undefined && span.data[GEN_AI_SYSTEM_FIELD] !== undefined) { + trace.metadata.modelProvider = String(span.data[GEN_AI_SYSTEM_FIELD]); + } +} + +function extractPromptData(span: Span, trace: AITrace) { + if (!span.data || trace.prompt) return; + + const promptMessages = span.data[GEN_AI_REQUEST_MESSAGES_FIELD]; + if (promptMessages === undefined) return; + + const system = span.data[GEN_AI_SYSTEM_INSTRUCTIONS_FIELD]; + const systemText = system !== undefined ? String(system) : undefined; + + try { + const messages = typeof promptMessages === "string" ? JSON.parse(promptMessages) : promptMessages; + if (Array.isArray(messages)) { + const normalizedMessages = messages.map((msg: { role: string; content: unknown }) => ({ + ...msg, + content: normalizeMessageContent(msg.content), + })); + trace.prompt = { messages: normalizedMessages, system: systemText }; + return; + } + } catch { + // fall through to the raw representation below + } + + trace.prompt = { messages: [{ role: "unknown", content: String(promptMessages) }], system: systemText }; +} + +function extractFinishReason(finishReasons: unknown): string | undefined { + if (Array.isArray(finishReasons) && finishReasons.length > 0) { + return String(finishReasons[0]); + } + if (typeof finishReasons === "string" && finishReasons) { + return finishReasons; + } + return undefined; +} + +function extractResponseData(span: Span, trace: AITrace, state: { finishReasonFromText: boolean }) { + if (!span.data) return; + + trace.response = trace.response || {}; + + // Spans arrive in tree pre-order, so for an agent root with several chat + // children the last span carrying a response wins — surfacing the latest + // model turn rather than the first. + const responseText = span.data[GEN_AI_RESPONSE_TEXT_FIELD]; + const text = Array.isArray(responseText) + ? responseText.map(String).join("") + : responseText + ? String(responseText) + : ""; + + const finishReason = extractFinishReason(span.data[GEN_AI_RESPONSE_FINISH_REASONS_FIELD]); + + if (text) { + trace.response.text = text; + // Re-pair the finish reason with the turn that produced the surfaced text, + // even overwriting one an earlier text-less turn contributed, so the text + // and its reason always come from the same turn. + trace.response.finishReason = finishReason; + state.finishReasonFromText = true; + } else if (finishReason !== undefined && !state.finishReasonFromText) { + // SDKs omit gen_ai.response.text when PII recording is off, and tool-call + // turns carry a reason with no text — still surface the reason, but never + // let it override one already paired with a text turn. + trace.response.finishReason = finishReason; + } + + const toolCalls = span.data[GEN_AI_RESPONSE_TOOL_CALLS_FIELD]; + if (toolCalls) { + try { + trace.response.toolCalls = typeof toolCalls === "string" ? JSON.parse(toolCalls) : (toolCalls as AIToolCall[]); + } catch { + // ignore unparseable tool call payloads + } + } +} + +function extractToolCallData(span: Span, trace: AITrace) { + if (!span.data) return; + + const toolName = span.data[GEN_AI_TOOL_NAME_FIELD]; + if (!toolName) return; + + const toolCallId = span.data[GEN_AI_TOOL_CALL_ID_FIELD]; + const toolCall: Partial = { + toolCallId: toolCallId ? String(toolCallId) : span.span_id, + toolName: String(toolName), + args: {} as Record, + }; + + const toolInput = span.data[GEN_AI_TOOL_INPUT_FIELD]; + if (toolInput) { + try { + toolCall.args = typeof toolInput === "string" ? JSON.parse(toolInput) : (toolInput as Record); + } catch { + toolCall.args = { input: toolInput }; + } + } + + const toolOutput = span.data[GEN_AI_TOOL_OUTPUT_FIELD]; + if (toolOutput) { + try { + toolCall.result = typeof toolOutput === "string" ? JSON.parse(toolOutput) : toolOutput; + } catch { + toolCall.result = String(toolOutput); + } + } + + trace.toolCalls.push(toolCall as AIToolCall); +} diff --git a/packages/spotlight/src/telemetry/ai/vercelAISDK.ts b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts new file mode 100644 index 000000000..af0ae337c --- /dev/null +++ b/packages/spotlight/src/telemetry/ai/vercelAISDK.ts @@ -0,0 +1,467 @@ +import { getAllSpansInTree } from "../store/helpers.ts"; +import type { AILibraryHandler, AIToolCall, AITrace, Span } from "../types.ts"; + +// https://ai-sdk.dev/docs/ai-sdk-core/telemetry +const AI_SPAN_OP_PREFIX = "gen_ai."; +const AI_OPERATION_ID_FIELD = "vercel.ai.operationId"; +const AI_OPERATION_NAME_FIELD = "operation.name"; + +// Tool call related constants (OpenTelemetry semantic conventions) +const AI_TOOL_CALL_OPERATION = "ai.toolCall"; +const AI_EXECUTE_TOOL_OPERATION = "gen_ai.execute_tool"; +const GEN_AI_TOOL_NAME_FIELD = "gen_ai.tool.name"; +const GEN_AI_TOOL_CALL_ID_FIELD = "gen_ai.tool.call.id"; +const GEN_AI_TOOL_INPUT_FIELD = "gen_ai.tool.input"; +const GEN_AI_TOOL_OUTPUT_FIELD = "gen_ai.tool.output"; + +const AI_MODEL_ID_FIELD = "vercel.ai.model.id"; +const AI_MODEL_PROVIDER_FIELD = "vercel.ai.model.provider"; +const AI_SETTINGS_MAX_RETRIES_FIELD = "vercel.ai.settings.maxRetries"; +const AI_SETTINGS_MAX_STEPS_FIELD = "vercel.ai.settings.maxSteps"; +const AI_TELEMETRY_FUNCTION_ID_FIELD = "vercel.ai.telemetry.functionId"; +const AI_TELEMETRY_METADATA_PREFIX = "vercel.ai.telemetry.metadata."; +const AI_PROMPT_FIELD = "vercel.ai.prompt"; +const AI_PROMPT_MESSAGES_FIELD = "gen_ai.request.messages"; +const AI_RESPONSE_FINISH_REASON_FIELD = "vercel.ai.response.finishReason"; +const GEN_AI_RESPONSE_FINISH_REASONS_FIELD = "gen_ai.response.finish_reasons"; +const AI_RESPONSE_TEXT_FIELD = "gen_ai.response.text"; +const AI_RESPONSE_TOOL_CALLS_FIELD = "vercel.ai.response.toolCalls"; + +// v1 operation names +const AI_STREAM_TEXT_OPERATION = "ai.streamText"; +const AI_GENERATE_TEXT_OPERATION = "ai.generateText"; + +// v2 operation names (OpenTelemetry semantic conventions) +const AI_INVOKE_AGENT_OPERATION = "gen_ai.invoke_agent"; +const AI_STREAM_TEXT_V2_OPERATION = "gen_ai.stream_text"; + +// v2 field names +const GEN_AI_PROMPT_FIELD = "gen_ai.prompt"; +const GEN_AI_FUNCTION_ID_FIELD = "gen_ai.function_id"; +const GEN_AI_REQUEST_MODEL_FIELD = "gen_ai.request.model"; +const GEN_AI_RESPONSE_MODEL_FIELD = "gen_ai.response.model"; +const GEN_AI_SYSTEM_FIELD = "gen_ai.system"; +const GEN_AI_OPERATION_NAME_FIELD = "gen_ai.operation.name"; + +const AI_USAGE_PROMPT_TOKENS_FIELD = "vercel.ai.usage.promptTokens"; +const AI_USAGE_COMPLETION_TOKENS_FIELD = "vercel.ai.usage.completionTokens"; +const GEN_AI_USAGE_INPUT_TOKENS_FIELD = "gen_ai.usage.input_tokens"; +const GEN_AI_USAGE_OUTPUT_TOKENS_FIELD = "gen_ai.usage.output_tokens"; + +const TOKEN_FIELDS = { + PROMPT: [AI_USAGE_PROMPT_TOKENS_FIELD, GEN_AI_USAGE_INPUT_TOKENS_FIELD], + COMPLETION: [AI_USAGE_COMPLETION_TOKENS_FIELD, GEN_AI_USAGE_OUTPUT_TOKENS_FIELD], +} as const; + +// Other constants +const DEFAULT_TRACE_NAME = "AI Interaction"; +const UNKNOWN_OPERATION = "N/A"; + +// Operation to badge mapping +const OPERATION_BADGE_MAP = new Map([ + [AI_STREAM_TEXT_OPERATION, "Stream Text"], + [AI_GENERATE_TEXT_OPERATION, "Generate Text"], + [AI_INVOKE_AGENT_OPERATION, "Invoke Agent"], + [AI_STREAM_TEXT_V2_OPERATION, "Stream Text"], +]); + +// Field mappings: first element is v2, subsequent are v1 alternatives +const AI_FIELD_MAPPINGS = { + MODEL_ID: [AI_MODEL_ID_FIELD, GEN_AI_REQUEST_MODEL_FIELD, GEN_AI_RESPONSE_MODEL_FIELD], + MODEL_PROVIDER: [AI_MODEL_PROVIDER_FIELD, GEN_AI_SYSTEM_FIELD], + FUNCTION_ID: [AI_TELEMETRY_FUNCTION_ID_FIELD, GEN_AI_FUNCTION_ID_FIELD], + PROMPT_TOKENS: [AI_USAGE_PROMPT_TOKENS_FIELD, GEN_AI_USAGE_INPUT_TOKENS_FIELD], + COMPLETION_TOKENS: [AI_USAGE_COMPLETION_TOKENS_FIELD, GEN_AI_USAGE_OUTPUT_TOKENS_FIELD], + PROMPT: [GEN_AI_PROMPT_FIELD, AI_PROMPT_FIELD], + OPERATION_ID: [AI_OPERATION_ID_FIELD, AI_OPERATION_NAME_FIELD, GEN_AI_OPERATION_NAME_FIELD], +} as const; + +/** + * Extracts a field value from span data, checking v1 fields first then v2 alternatives. + */ +function extractFieldFromSpan( + data: Record, + fieldKey: keyof typeof AI_FIELD_MAPPINGS, +): unknown | undefined { + const fields = AI_FIELD_MAPPINGS[fieldKey]; + for (const field of fields) { + if (data[field] !== undefined) { + return data[field]; + } + } + return undefined; +} + +export const vercelAISDKHandler: AILibraryHandler = { + id: "vercel-ai-sdk", + name: "Vercel AI SDK", + + canHandleSpan: (span: Span): boolean => { + return !!span.op?.toLowerCase().startsWith(AI_SPAN_OP_PREFIX); + }, + + extractRootSpans: (spans: Span[]): Span[] => { + const resultRoots: Span[] = []; + + const findAndCollectAIRoots = (spansToSearch: Span[]) => { + for (const currentSpan of spansToSearch) { + if (currentSpan.op?.toLowerCase().startsWith(AI_SPAN_OP_PREFIX)) { + resultRoots.push(currentSpan); + } else if (currentSpan.children?.length) { + findAndCollectAIRoots(currentSpan.children); + } + } + }; + + findAndCollectAIRoots(spans); + return resultRoots; + }, + + processTrace: (rootSpan: Span): AITrace => { + const allSpans = getAllSpansInTree(rootSpan); + const tokenUsage = extractTokenUsage(allSpans); + const operationInfo = determineOperation(allSpans, rootSpan); + const traceName = determineTraceName(rootSpan, operationInfo); + + const durationMs = rootSpan.timestamp - rootSpan.start_timestamp; + + // Initialize the trace with basic properties + const trace: AITrace = { + id: rootSpan.span_id, + name: traceName, + operation: operationInfo.operation, + timestamp: rootSpan.start_timestamp, + durationMs, + tokensDisplay: formatTokensDisplay(tokenUsage.promptTokens, tokenUsage.completionTokens), + promptTokens: tokenUsage.promptTokens, + completionTokens: tokenUsage.completionTokens, + hasToolCall: operationInfo.hasToolCall, + rawSpan: rootSpan, + metadata: { + metadata: {}, + }, + toolCalls: [], + }; + + // Parse detailed span data + parseSpanData(allSpans, trace); + + return trace; + }, + + getDisplayTitle: (trace: AITrace): string => { + return trace.name || trace.operation || `AI Trace ${trace.id.substring(0, 8)}`; + }, + + getTypeBadge: (trace: AITrace): string => { + if (trace.hasToolCall) { + return "Tool-Call"; + } + + return OPERATION_BADGE_MAP.get(trace.operation) ?? trace.operation.replace(/^(ai\.|gen_ai\.)/, ""); + }, + + getTokensDisplay: (trace: AITrace): string => { + return trace.tokensDisplay; + }, +}; + +function extractTokenUsage(spans: Span[]): { promptTokens?: number; completionTokens?: number } { + let promptTokens: number | undefined; + let completionTokens: number | undefined; + + for (const span of spans) { + if (!span.data) continue; + + if (promptTokens === undefined) { + for (const field of TOKEN_FIELDS.PROMPT) { + const value = span.data[field]; + if (value !== undefined) { + promptTokens = Number(value); + break; + } + } + } + + if (completionTokens === undefined) { + for (const field of TOKEN_FIELDS.COMPLETION) { + const value = span.data[field]; + if (value !== undefined) { + completionTokens = Number(value); + break; + } + } + } + + if (promptTokens !== undefined && completionTokens !== undefined) { + break; + } + } + + return { promptTokens, completionTokens }; +} + +function determineOperation( + spans: Span[], + rootSpan: Span, +): { operation: string; hasToolCall: boolean; toolCallName?: string } { + let operation = UNKNOWN_OPERATION; + let hasToolCall = false; + let toolCallName: string | undefined; + let foundToolCallAsOperationId = false; + + for (const span of spans) { + if (span.op === AI_TOOL_CALL_OPERATION || span.op === AI_EXECUTE_TOOL_OPERATION) { + hasToolCall = true; + } + + if (!span.data) continue; + + const operationId = extractFieldFromSpan(span.data, "OPERATION_ID") as string | undefined; + + // Handle tool call operation + if (operationId === AI_TOOL_CALL_OPERATION) { + operation = AI_TOOL_CALL_OPERATION; + foundToolCallAsOperationId = true; + hasToolCall = true; + + if (span.data[GEN_AI_TOOL_NAME_FIELD]) { + toolCallName = String(span.data[GEN_AI_TOOL_NAME_FIELD]); + } + } + + // Handle other operations (prioritize root and direct children) + if (!foundToolCallAsOperationId && operationId && operation === UNKNOWN_OPERATION) { + const isHighPriority = + span.span_id === rootSpan.span_id || rootSpan.children?.some(child => child.span_id === span.span_id); + + if (isHighPriority) { + operation = operationId; + } + } + } + + // Fallback logic + if (operation === UNKNOWN_OPERATION) { + if (rootSpan.op) { + operation = rootSpan.op; + } else if (rootSpan.description) { + operation = rootSpan.description; + } + } + + return { operation, hasToolCall, toolCallName }; +} + +function determineTraceName(rootSpan: Span, operationInfo: { toolCallName?: string }): string { + if (operationInfo.toolCallName) { + return operationInfo.toolCallName; + } + + return rootSpan.description || rootSpan.op || DEFAULT_TRACE_NAME; +} + +function formatTokensDisplay(promptTokens?: number, completionTokens?: number): string { + if (promptTokens !== undefined && completionTokens !== undefined) { + return `${promptTokens} / ${completionTokens}`; + } + if (promptTokens !== undefined) { + return `${promptTokens} / ?`; + } + if (completionTokens !== undefined) { + return `? / ${completionTokens}`; + } + return UNKNOWN_OPERATION; +} + +/** + * Normalizes message content from v1 (string) or v2 (array of content blocks) format to a string. + * v1: content = "text string" + * v2: content = [{ type: "text", text: "text string" }] + * + * Only extracts text content blocks. Other block types (image, file, tool-call) + * would require UI changes to display properly and are filtered out for now. + */ +function normalizeMessageContent(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + // Only extract text content - images/files would need UI changes to display + return content + .filter( + (block): block is { type: string; text: string } => block.type === "text" && typeof block.text === "string", + ) + .map(block => block.text) + .join(""); + } + return ""; +} + +function parseSpanData(spans: Span[], trace: AITrace) { + for (const span of spans) { + if (!span.data) continue; + + extractAIMetadata(span, trace); + extractTelemetryMetadata(span, trace); + extractPromptData(span, trace); + extractResponseData(span, trace); + extractToolCallData(span, trace); + } +} + +function extractAIMetadata(span: Span, trace: AITrace) { + if (!span.data) return; + + // Model ID + if (trace.metadata.modelId === undefined) { + const modelId = extractFieldFromSpan(span.data, "MODEL_ID"); + if (modelId !== undefined) { + trace.metadata.modelId = String(modelId); + } + } + + // Model Provider + if (trace.metadata.modelProvider === undefined) { + const provider = extractFieldFromSpan(span.data, "MODEL_PROVIDER"); + if (provider !== undefined) { + trace.metadata.modelProvider = String(provider); + } + } + + // Function ID + if (trace.metadata.functionId === undefined) { + const functionId = extractFieldFromSpan(span.data, "FUNCTION_ID"); + if (functionId !== undefined) { + trace.metadata.functionId = String(functionId); + } + } + + if (span.data[AI_SETTINGS_MAX_RETRIES_FIELD]) { + trace.metadata.maxRetries = Number(span.data[AI_SETTINGS_MAX_RETRIES_FIELD]); + } + + if (span.data[AI_SETTINGS_MAX_STEPS_FIELD]) { + trace.metadata.maxSteps = Number(span.data[AI_SETTINGS_MAX_STEPS_FIELD]); + } + + // Token usage - use === undefined to allow 0 values + if (trace.metadata.promptTokens === undefined) { + const promptTokens = extractFieldFromSpan(span.data, "PROMPT_TOKENS"); + if (promptTokens !== undefined) { + trace.metadata.promptTokens = Number(promptTokens); + } + } + + if (trace.metadata.completionTokens === undefined) { + const completionTokens = extractFieldFromSpan(span.data, "COMPLETION_TOKENS"); + if (completionTokens !== undefined) { + trace.metadata.completionTokens = Number(completionTokens); + } + } +} + +function extractTelemetryMetadata(span: Span, trace: AITrace) { + if (!span.data) return; + + for (const [key, value] of Object.entries(span.data)) { + if (key.startsWith(AI_TELEMETRY_METADATA_PREFIX)) { + const metadataKey = key.replace(AI_TELEMETRY_METADATA_PREFIX, ""); + trace.metadata.metadata[metadataKey] = value; + } + } +} + +function extractPromptData(span: Span, trace: AITrace) { + if (!span.data) return; + + const promptField = extractFieldFromSpan(span.data, "PROMPT"); + if (promptField && !trace.prompt) { + try { + const parsed = JSON.parse(String(promptField)); + // Normalize message content from v2 array format to string + if (parsed.messages && Array.isArray(parsed.messages)) { + parsed.messages = parsed.messages.map((msg: { role: string; content: unknown }) => ({ + ...msg, + content: normalizeMessageContent(msg.content), + })); + } + trace.prompt = parsed; + } catch { + trace.prompt = { messages: [{ role: "unknown", content: String(promptField) }] }; + } + } + + const promptMessages = span.data[AI_PROMPT_MESSAGES_FIELD]; + if (promptMessages && !trace.prompt) { + try { + const messages = JSON.parse(String(promptMessages)); + // Normalize message content from v2 array format to string + const normalizedMessages = messages.map((msg: { role: string; content: unknown }) => ({ + ...msg, + content: normalizeMessageContent(msg.content), + })); + trace.prompt = { messages: normalizedMessages }; + } catch { + trace.prompt = { messages: [{ role: "unknown", content: String(promptMessages) }] }; + } + } +} + +function extractResponseData(span: Span, trace: AITrace) { + if (!span.data) return; + + trace.response = trace.response || {}; + + const finishReason = span.data[AI_RESPONSE_FINISH_REASON_FIELD]; + if (finishReason) { + trace.response.finishReason = String(finishReason); + } + + const finishReasons = span.data[GEN_AI_RESPONSE_FINISH_REASONS_FIELD]; + if (!trace.response.finishReason && Array.isArray(finishReasons) && finishReasons.length > 0) { + trace.response.finishReason = String(finishReasons[0]); + } + + const responseText = span.data[AI_RESPONSE_TEXT_FIELD]; + if (responseText) { + trace.response.text = String(responseText); + } + + const toolCalls = span.data[AI_RESPONSE_TOOL_CALLS_FIELD]; + if (toolCalls) { + trace.response.toolCalls = JSON.parse(String(toolCalls)); + } +} + +function extractToolCallData(span: Span, trace: AITrace) { + if (!span.data) return; + + const toolName = span.data[GEN_AI_TOOL_NAME_FIELD]; + if (!toolName) return; + + const toolCallId = span.data[GEN_AI_TOOL_CALL_ID_FIELD]; + const toolCall: Partial = { + toolCallId: toolCallId ? String(toolCallId) : span.span_id, + toolName: String(toolName), + args: {} as Record, + }; + + const toolInput = span.data[GEN_AI_TOOL_INPUT_FIELD]; + if (toolInput) { + try { + toolCall.args = typeof toolInput === "string" ? JSON.parse(toolInput) : (toolInput as Record); + } catch { + toolCall.args = { input: toolInput }; + } + } + + const toolOutput = span.data[GEN_AI_TOOL_OUTPUT_FIELD]; + if (toolOutput) { + try { + toolCall.result = typeof toolOutput === "string" ? JSON.parse(toolOutput) : toolOutput; + } catch { + toolCall.result = String(toolOutput); + } + } + + trace.toolCalls.push(toolCall as AIToolCall); +} diff --git a/packages/spotlight/src/telemetry/attachments.spec.ts b/packages/spotlight/src/telemetry/attachments.spec.ts new file mode 100644 index 000000000..69f285a65 --- /dev/null +++ b/packages/spotlight/src/telemetry/attachments.spec.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "vitest"; +import { createTelemetry } from "../sdk/core.ts"; + +describe("parsed protocol attachments", () => { + test.each([ + ["binary bytes", new Uint8Array([0, 10, 255]), "application/octet-stream", "AAr/"], + ["binary string", "héllo", "application/octet-stream", "aMOpbGxv"], + ["text bytes", new TextEncoder().encode("héllo\n世界"), "text/plain", "héllo\n世界"], + ["text string", "héllo\n世界", "text/plain", "héllo\n世界"], + ["text with byte order mark", new Uint8Array([239, 187, 191, 65]), "text/plain", "\uFEFFA"], + ] as const)( + "associates %s with its event and retains canonical immutable data", + async (_, payload, contentType, data) => { + const telemetry = createTelemetry(); + const input = [ + {}, + [ + [{ type: "event" }, { event_id: "attached-event", message: "hello", timestamp: 1 }], + [{ type: "attachment", filename: "file", content_type: contentType }, payload], + ], + ]; + const original = structuredClone(input); + const result = await telemetry.ingestEnvelope(input); + expect(result.status).toBe("processed"); + expect(telemetry.getEvent("attached-event")?.attachments).toMatchObject([{ data }]); + expect(telemetry.getEnvelopes()[0][1][1][1]).toEqual({ data }); + expect(Object.isFrozen(telemetry.getEnvelopes()[0][1][1][1])).toBe(true); + expect(input).toEqual(original); + telemetry.dispose(); + }, + ); + + test("retains opaque unsupported items without discarding their accompanying event", async () => { + const telemetry = createTelemetry(); + const result = await telemetry.ingestEnvelope([ + {}, + [ + [{ type: "event" }, { event_id: "event", timestamp: 1 }], + [{ type: "future_binary" }, new Uint8Array([0, 255])], + [{ type: "future_text" }, "hello"], + ], + ]); + expect(result.status).toBe("processed"); + expect(telemetry.getEvent("event")).toBeDefined(); + expect( + telemetry + .getEnvelopes()[0][1] + .slice(1) + .map(item => item[1]), + ).toEqual([{ data: "AP8=" }, { data: "aGVsbG8=" }]); + telemetry.dispose(); + }); + + test("continues to reject primitive payloads for supported telemetry items", async () => { + const telemetry = createTelemetry(); + const result = await telemetry.ingestEnvelope([{}, [[{ type: "event" }, "not a parsed event"]]]); + expect(result.status).toBe("invalid"); + expect(telemetry.getEnvelopes()).toEqual([]); + telemetry.dispose(); + }); +}); diff --git a/packages/spotlight/src/telemetry/client.spec.ts b/packages/spotlight/src/telemetry/client.spec.ts new file mode 100644 index 000000000..3a39e23c8 --- /dev/null +++ b/packages/spotlight/src/telemetry/client.spec.ts @@ -0,0 +1,399 @@ +// @vitest-environment node +import { afterEach, expect, test, vi } from "vitest"; +import * as core from "./index.ts"; + +function spans(id: string, traceId = "trace", count = 1) { + return [ + { __spotlight_envelope_id: id, sdk: { name: "sentry.javascript.node", version: "11" } }, + [ + [ + { type: "span" }, + { + version: 2, + items: Array.from({ length: count }, (_, index) => ({ + trace_id: traceId, + span_id: `${id}-${index}`, + name: "SELECT 1", + start_timestamp: 1, + end_timestamp: 1.125, + attributes: { "sentry.op": { value: "db.query", type: "string" } }, + })), + }, + ], + ], + ]; +} +afterEach(() => vi.useRealTimers()); + +test("canonical reads have consistent filters and explicit summary units", async () => { + const client = core.createTelemetry(); + const result = await client.ingestEnvelope(spans("one")); + expect(result).toMatchObject({ status: "processed", envelopeId: "one", diagnostics: [] }); + expect(client.getQuerySummaries({ traceId: "trace" })).toMatchObject([ + { count: 1, totalDurationMs: 125, averageDurationMs: 125 }, + ]); + expect(client.getTrace("trace")?.spans.size).toBe(1); + expect(client.getSpans({ traceId: "absent" })).toEqual([]); + expect(client.getQuerySummaries({ traceId: "absent" })).toEqual([]); + expect(client).not.toHaveProperty("store"); + client.dispose(); +}); + +test("subscribers see exactly one coherent update per ingestion", async () => { + const client = core.createTelemetry(); + const seen: number[] = []; + client.subscribe(snapshot => { + expect(snapshot.events).toHaveLength(3); + expect(snapshot.traces).toHaveLength(1); + expect(client.getQuerySummaries()[0].count).toBe(3); + seen.push(snapshot.revision); + }); + const initial = client.getSnapshot(); + await client.ingestEnvelope(spans("three", "trace", 3)); + expect(seen).toHaveLength(1); + expect(client.getSnapshot()).toBe(client.getSnapshot()); + expect(initial.events).toEqual([]); + client.dispose(); +}); + +test("returned events, nested maps and earlier snapshots cannot mutate engine state", async () => { + const client = core.createTelemetry(); + await client.ingestEnvelope(spans("one")); + const snapshot = client.getSnapshot(); + const trace = client.getTrace("trace")!; + expect(() => (trace.spans as Map).clear()).toThrow(); + expect(() => { + (trace.spans.values().next().value as { description: string }).description = "changed"; + }).toThrow(); + expect(() => (client.getEvents() as unknown[]).pop()).toThrow(); + trace.spans.forEach((_value, _key, map) => expect(() => (map as Map).clear()).toThrow()); + await client.ingestEnvelope(spans("two")); + expect(snapshot.traces[0].spans.size).toBe(1); + expect(client.getTrace("trace")?.spans.size).toBe(2); + expect(client.getQuerySummaries()[0].description).toBe("SELECT 1"); + client.dispose(); +}); + +test("snapshots preserve own prototype-named JSON fields without inheriting their values", async () => { + const client = core.createTelemetry(); + const input = JSON.parse( + '[{"__spotlight_envelope_id":"proto"},[[{"type":"event"},{"event_id":"proto","timestamp":1,"extra":{"__proto__":{"unexpected":true},"kept":1}}]]]', + ); + expect(await client.ingestEnvelope(input)).toMatchObject({ status: "processed" }); + const extra = client.getEvent("proto")!.extra!; + expect(Object.hasOwn(extra, "__proto__")).toBe(true); + expect(extra.unexpected).toBeUndefined(); + expect(JSON.stringify(client.getEnvelope("proto"))).toBe(JSON.stringify(input)); + expect(() => { + (extra.__proto__ as { unexpected: boolean }).unexpected = false; + }).toThrow(); + client.dispose(); +}); + +test("invalid and unsupported envelopes return diagnostics without changing the snapshot", async () => { + const client = core.createTelemetry(); + const initial = client.getSnapshot(); + expect(await client.ingestEnvelope(null)).toMatchObject({ + status: "invalid", + diagnostics: [{ code: "invalid_envelope" }], + }); + expect(client.getSnapshot()).toBe(initial); + expect(await client.ingestEnvelope([{}, [[{ type: "future" }, {}]]])).toMatchObject({ + status: "ignored", + reason: "unsupported_items", + }); + expect(await client.ingestEnvelope(spans("duplicate"))).toMatchObject({ status: "processed" }); + const snapshot = client.getSnapshot(); + expect(await client.ingestEnvelope(spans("duplicate"))).toMatchObject({ status: "ignored", reason: "duplicate" }); + expect(client.getSnapshot()).toBe(snapshot); + client.dispose(); +}); + +test("invalid parsed input reports the same diagnostic to the observer and caller", async () => { + const diagnostics: string[] = []; + const client = core.createTelemetry({ onDiagnostic: diagnostic => diagnostics.push(diagnostic.code) }); + const result = await client.ingestEnvelope(null); + expect(diagnostics).toEqual(["invalid_envelope"]); + expect(result.diagnostics.map(diagnostic => diagnostic.code)).toEqual(diagnostics); + client.dispose(); +}); + +test("a rejected envelope waits for sibling enrichment and cannot poison later events", async () => { + let release!: () => void; + let entered!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + const ready = new Promise(resolve => { + entered = resolve; + }); + let enrichments = 0; + const client = core.createTelemetry({ + enrichStacktrace: async event => { + enrichments++; + if (enrichments === 1) { + entered(); + await gate; + } + event.exception.values![0].value = enrichments === 1 ? "invalid-cached" : "valid-context"; + }, + }); + const error = { event_id: "same", timestamp: 1, exception: { values: [{ type: "Error", value: "original" }] } }; + const pending = client.ingestEnvelope([ + {}, + [ + [{ type: "event" }, error], + [{ type: "log" }, { items: [null] }], + ], + ]); + await ready; + let finished = false; + void pending.then(() => { + finished = true; + }); + await new Promise(resolve => setImmediate(resolve)); + expect(finished).toBe(false); + release(); + expect(await pending).toMatchObject({ status: "invalid" }); + expect(client.getEvents()).toEqual([]); + await client.ingestEnvelope([{}, [[{ type: "event" }, error]]]); + expect(enrichments).toBe(2); + expect(client.getErrors()[0].exception.values?.[0].value).toBe("valid-context"); + client.dispose(); +}); + +test("malformed spans have diagnostics while mixed valid spans still process", async () => { + const client = core.createTelemetry(); + expect(await client.ingestEnvelope([{}, [[{ type: "span" }, { version: 2, items: [{}] }]]])).toMatchObject({ + status: "invalid", + diagnostics: [{ code: "invalid_span" }], + }); + const input = spans("partial"); + (input[1][0][1] as { items: unknown[] }).items.push({}); + expect(await client.ingestEnvelope(input)).toMatchObject({ + status: "processed", + diagnostics: [{ code: "invalid_span" }], + }); + expect(client.getSpans()).toHaveLength(1); + client.dispose(); +}); + +test("retention preserves identities of profile spans derived from retained envelopes", async () => { + const { readFileSync } = await import("node:fs"); + const { createNodeTelemetry } = await import("../sdk/node.ts"); + const client = createNodeTelemetry({ retention: { maxEnvelopes: 3 } }); + await client.ingestEnvelope(spans("evict-me")); + for (const file of ["transaction.txt", "profile_chunk_1.txt"]) { + const body = readFileSync(new URL(`../../_fixtures/continuous_profiling/${file}`, import.meta.url)); + const result = await client.ingest({ body }); + expect(result.status).toBe("processed"); + } + const traceId = "f1e2d3c4b5a6978012345678901234ab"; + const ids = client.getSpans({ traceId }).map(span => span.span_id); + expect(client.getSpans({ traceId }).some(span => span.tags?.source === "profile")).toBe(true); + await client.ingestEnvelope(spans("new")); + expect(client.getSpans({ traceId }).map(span => span.span_id)).toEqual(ids); + client.dispose(); +}); + +test("count retention evicts related events, traces, logs and profiles coherently", async () => { + const client = core.createTelemetry({ retention: { maxEnvelopes: 1 } }); + await client.ingestEnvelope([ + { __spotlight_envelope_id: "old" }, + [ + [{ type: "log" }, { items: [{ body: "old", timestamp: 1, trace_id: "old", attributes: {} }] }], + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "old", + span_id: "old", + name: "SELECT old", + start_timestamp: 1, + end_timestamp: 2, + attributes: { "sentry.op": { value: "db" } }, + }, + ], + }, + ], + ], + ]); + await client.ingestEnvelope(spans("new", "new")); + expect(client.getEnvelopes()).toHaveLength(1); + expect(client.getTrace("old")).toBeUndefined(); + expect(client.getLogs({ traceId: "old" })).toEqual([]); + expect(client.getEvents()).toHaveLength(1); + expect(client.getQuerySummaries()).toMatchObject([{ description: "SELECT 1", count: 1 }]); + client.dispose(); +}); + +test("retention replay preserves generated log IDs and does not repeat enrichment", async () => { + let enrichments = 0; + const client = core.createTelemetry({ + retention: { maxEnvelopes: 2 }, + enrichStacktrace: async () => { + enrichments++; + }, + }); + await client.ingestEnvelope(spans("first")); + await client.ingestEnvelope([ + { __spotlight_envelope_id: "retained" }, + [ + [{ type: "log" }, { items: [{ body: "hello", timestamp: 1, attributes: {} }] }], + [ + { type: "event" }, + { event_id: "error", timestamp: 1, exception: { values: [{ type: "Error", value: "error" }] } }, + ], + ], + ]); + const id = client.getLogs()[0].id; + await client.ingestEnvelope(spans("last")); + expect(client.getLogs()[0].id).toBe(id); + expect(enrichments).toBe(1); + expect(client.getErrors()).toHaveLength(1); + client.dispose(); +}); + +test("duplicate error IDs have one deterministic enrichment owner across replay", async () => { + let release!: () => void; + let entered!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + const ready = new Promise(resolve => { + entered = resolve; + }); + let calls = 0; + const client = core.createTelemetry({ + retention: { maxEnvelopes: 2 }, + enrichStacktrace: async () => { + calls++; + if (calls === 1) { + entered(); + await gate; + } + }, + }); + await client.ingestEnvelope(spans("old")); + const pending = client.ingestEnvelope([ + { __spotlight_envelope_id: "keep" }, + ["first", "second"].map(value => [ + { type: "event" }, + { event_id: "same", timestamp: 1, exception: { values: [{ type: "Error", value }] } }, + ]), + ]); + await ready; + release(); + await pending; + expect(calls).toBe(1); + expect(client.getErrors()[0].exception.values?.[0].value).toBe("first"); + await client.ingestEnvelope(spans("new")); + expect(client.getErrors()[0].exception.values?.[0].value).toBe("first"); + expect(calls).toBe(1); + client.dispose(); +}); + +test("evicting an error owner enriches the retained duplicate from its own exception", async () => { + const seen: string[] = []; + const client = core.createTelemetry({ + retention: { maxEnvelopes: 3 }, + enrichStacktrace: async event => { + const exception = event.exception.values![0]; + seen.push(exception.value!); + exception.value = `enriched:${exception.value}`; + }, + }); + for (const id of ["old", "retained"]) { + await client.ingestEnvelope([ + { __spotlight_envelope_id: id }, + [[{ type: "event" }, { event_id: "same", timestamp: 1, exception: { values: [{ value: id }] } }]], + ]); + if (id === "old") await client.ingestEnvelope(spans("middle")); + } + expect(client.getErrors()[0].exception.values![0].value).toBe("enriched:old"); + await client.ingestEnvelope(spans("new")); + expect(client.getErrors()[0].exception.values![0].value).toBe("enriched:retained"); + expect(seen).toEqual(["old", "retained"]); + await client.ingestEnvelope(spans("next")); + expect(seen).toEqual(["old", "retained"]); + client.dispose(); +}); + +test("SDK last-seen times do not advance when retained envelopes replay", async () => { + vi.useFakeTimers(); + const client = core.createTelemetry({ retention: { maxEnvelopes: 2 } }); + for (const [index, name] of ["old", "keep", "new"].entries()) { + vi.setSystemTime(1000 * (index + 1)); + await client.ingestEnvelope([ + { __spotlight_envelope_id: name }, + [[{ type: "event" }, { event_id: name, timestamp: 1, sdk: { name, version: "1" } }]], + ]); + } + expect(client.getSdks().find(sdk => sdk.name === "keep")?.lastSeen).toBe(2000); + client.dispose(); +}); + +test("age retention expires while idle and disposes its timer", async () => { + vi.useFakeTimers(); + const client = core.createTelemetry({ retention: { maxAgeMs: 1000 } }); + await client.ingestEnvelope(spans("old")); + await vi.advanceTimersByTimeAsync(1000); + expect(client.getTraces()).toEqual([]); + expect(client.getEnvelopes()).toEqual([]); + expect(vi.getTimerCount()).toBe(0); + await client.ingestEnvelope(spans("new")); + client.dispose(); + expect(vi.getTimerCount()).toBe(0); + expect(await client.ingestEnvelope(spans("closed"))).toMatchObject({ status: "ignored", reason: "disposed" }); +}); + +test("reset invalidates active and queued work while allowing immediate new ingestion", async () => { + let release!: () => void; + const gate = new Promise(resolve => { + release = resolve; + }); + let entered!: () => void; + const ready = new Promise(resolve => { + entered = resolve; + }); + const client = core.createTelemetry({ + enrichStacktrace: () => { + entered(); + return gate; + }, + }); + const active = client.ingestEnvelope([ + { __spotlight_envelope_id: "error" }, + [[{ type: "event" }, { event_id: "error", timestamp: 1, exception: { values: [] } }]], + ]); + await ready; + const queued = client.ingestEnvelope(spans("queued")); + client.reset(); + await client.ingestEnvelope(spans("new")); + expect(client.getEvents()).toHaveLength(1); + release(); + expect(await active).toMatchObject({ status: "ignored", reason: "reset" }); + expect(await queued).toMatchObject({ status: "ignored", reason: "reset" }); + expect(client.getEvents()).toHaveLength(1); + expect(client.getErrors()).toEqual([]); + client.dispose(); +}); + +test("subscribers can reset without a later notification restoring stale state", async () => { + const client = core.createTelemetry(); + let reset = false; + const observed: number[] = []; + client.subscribe(snapshot => { + if (!reset && snapshot.events.length) { + reset = true; + client.reset(); + } + }); + client.subscribe(snapshot => observed.push(snapshot.events.length)); + await client.ingestEnvelope(spans("old")); + expect(client.getTraces()).toEqual([]); + expect(observed).toEqual([0]); + client.dispose(); +}); diff --git a/packages/spotlight/src/telemetry/constants/performance.ts b/packages/spotlight/src/telemetry/constants/performance.ts new file mode 100644 index 000000000..7d9fd63a6 --- /dev/null +++ b/packages/spotlight/src/telemetry/constants/performance.ts @@ -0,0 +1,60 @@ +export const PERFORMANCE_SCORE_PROFILES = { + profiles: [ + { + name: "Chrome", + scoreComponents: [ + { measurement: "fcp", weight: 0.15, p10: 900.0, p50: 1600.0, optional: false }, + { measurement: "lcp", weight: 0.3, p10: 1200.0, p50: 2400.0, optional: false }, + { measurement: "fid", weight: 0.3, p10: 100.0, p50: 300.0, optional: true }, + { measurement: "cls", weight: 0.15, p10: 0.1, p50: 0.25, optional: false }, + { measurement: "ttfb", weight: 0.1, p10: 200.0, p50: 400.0, optional: false }, + ], + condition: { op: "eq", name: "event.contexts.browser.name", value: "Chrome" }, + }, + // TODO: Currently not getting browser data in events, so made these comments and sticked to chrome always. + // { + // "name": "Firefox", + // "scoreComponents": [ + // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, + // {"measurement": "lcp", "weight": 0.0, "p10": 1200.0, "p50": 2400.0, "optional": false}, + // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, + // {"measurement": "cls", "weight": 0.0, "p10": 0.1, "p50": 0.25, "optional": false}, + // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} + // ], + // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Firefox"} + // }, + // { + // "name": "Safari", + // "scoreComponents": [ + // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, + // {"measurement": "lcp", "weight": 0.0, "p10": 1200.0, "p50": 2400.0, "optional": false}, + // {"measurement": "fid", "weight": 0.0, "p10": 100.0, "p50": 300.0, "optional": true}, + // {"measurement": "cls", "weight": 0.0, "p10": 0.1, "p50": 0.25, "optional": false}, + // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} + // ], + // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Safari"} + // }, + // { + // "name": "Edge", + // "scoreComponents": [ + // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, + // {"measurement": "lcp", "weight": 0.30, "p10": 1200.0, "p50": 2400.0, "optional": false}, + // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, + // {"measurement": "cls", "weight": 0.15, "p10": 0.1, "p50": 0.25, "optional": false}, + // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} + // ], + // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Edge"} + // }, + // { + // "name": "Opera", + // "scoreComponents": [ + // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, + // {"measurement": "lcp", "weight": 0.30, "p10": 1200.0, "p50": 2400.0, "optional": false}, + // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, + // {"measurement": "cls", "weight": 0.15, "p10": 0.1, "p50": 0.25, "optional": false}, + // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} + // ], + // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Opera"} + // } + ], +}; diff --git a/packages/spotlight/src/telemetry/constants/sentry.ts b/packages/spotlight/src/telemetry/constants/sentry.ts new file mode 100644 index 000000000..1bedc69f2 --- /dev/null +++ b/packages/spotlight/src/telemetry/constants/sentry.ts @@ -0,0 +1,14 @@ +export const RAW_TYPES = new Set(["attachment", "replay_video", "statsd"]); +export const ERROR_EVENT_TYPES = new Set(["event", "error"]); +export const TRACE_EVENT_TYPES = new Set(["transaction"]); +export const PROFILE_EVENT_TYPES = new Set(["profile", "profile_chunk"]); +export const LOG_EVENT_TYPES = new Set(["log"]); +export const FEEDBACK_EVENT_TYPES = new Set(["feedback"]); +export const ATTACHMENT_EVENT_TYPES = new Set(["attachment"]); +export const SUPPORTED_EVENT_TYPES = new Set([ + ...ERROR_EVENT_TYPES, + ...TRACE_EVENT_TYPES, + ...PROFILE_EVENT_TYPES, + ...LOG_EVENT_TYPES, + ...FEEDBACK_EVENT_TYPES, +]); diff --git a/packages/spotlight/src/telemetry/data/profiles.ts b/packages/spotlight/src/telemetry/data/profiles.ts new file mode 100644 index 000000000..4334f7bbd --- /dev/null +++ b/packages/spotlight/src/telemetry/data/profiles.ts @@ -0,0 +1,215 @@ +import { log } from "../logger.ts"; +import type { SentryProfileWithTraceMeta } from "../store/types.ts"; +import type { Span, Trace } from "../types.ts"; +import { getFunctionNameFromFrame } from "../utils/profileUtils.ts"; +import { compareSpans } from "../utils/traces.ts"; + +/** Non-cryptographic FNV-1a identity for a derived profile frame, stable across replays. */ +function profileSpanId(parts: unknown[]): string { + let hash = 0x6c62272e07bb014262b821756295c58dn; + for (const byte of new TextEncoder().encode(JSON.stringify(parts))) { + hash = BigInt.asUintN(128, (hash ^ BigInt(byte)) * 0x1000000000000000000013bn); + } + return hash.toString(16).padStart(32, "0"); +} + +/** + * Groups consequent spans with the same description and op into a single span per each level. + * Essentially a BFS traversal of the spans tree. + * @param spans Span[] A list of spans to consolidate, sorted by their start_timestamp + * @returns Span[] A list of spans with the same description and op consolidated into a single span + */ +function consolidateSpans(trace: Trace, spans: Span[]): Span[] { + const consolidatedSpans: Span[] = []; + let lastSpan = spans[0]; + let spanIdx = 1; + while (spanIdx < spans.length + 1) { + const span = spans[spanIdx]; + spanIdx += 1; + if (span && span.description === lastSpan.description && span.op === lastSpan.op) { + // Require the spans are sorted by start_timestamp + lastSpan.timestamp = span.timestamp; + if (span.children) { + if (lastSpan.children) { + for (const child of span.children) { + lastSpan.children.push(child); + } + lastSpan.children.sort(compareSpans); + } else { + lastSpan.children = span.children; + } + } + } else { + // Discard very short spans + if (lastSpan.timestamp - lastSpan.start_timestamp > 0) { + if (lastSpan.children) { + lastSpan.children = consolidateSpans(trace, lastSpan.children); + } + consolidatedSpans.push(lastSpan); + trace.spans.set(lastSpan.span_id, lastSpan); + } + lastSpan = span; + } + } + + return consolidatedSpans; +} + +// This is done per platform right now but we may want to make it use +// SDK or something more specific, especially for JS as `javascript` may +// mean browser, node, etc. +const SENTRY_FRAME_FILTER_PER_PLATFORM: Record< + string, + (this: SentryProfileWithTraceMeta["frames"], frameIdx: number) => boolean | undefined +> = { + python: function (frameIdx) { + return this[frameIdx].module?.startsWith("sentry_sdk."); + }, + javascript: function (frameIdx) { + const frame = this[frameIdx]; + const module = frame.module; + if (module) { + return module.startsWith("@sentry") || module.startsWith("@opentelemetry.instrumentation"); + } + // This one below is to match things like `http://localhost:3000/node_modules/.vite/deps/@sentry_react.js?v=6942e78f` etc. + return frame.abs_path ? /\/node_modules\/.*\/@(sentry|opentelemetry)[^a-z0-9]/.test(frame.abs_path) : false; + }, +}; + +export function getSpansFromProfile( + trace: Trace, + profile: SentryProfileWithTraceMeta, + parent_span_id: string | undefined, + startTs: number, + endTs: number, + threadIds: Set, +): Span[] { + threadIds.add(profile.active_thread_id); + + const sentryFrameFilter = profile.platform && SENTRY_FRAME_FILTER_PER_PLATFORM[profile.platform]; + // Try to fill in the gaps from profile data + const fillerSpans: Span[] = []; + for (let sampleIdx = 0; sampleIdx < profile.samples.length; sampleIdx++) { + const sample = profile.samples[sampleIdx]; + if (sample.thread_id && !threadIds.has(sample.thread_id)) { + continue; + } + const sampleTs = sample.start_timestamp; + if (sampleTs < startTs || sampleTs > endTs) { + continue; + } + const nextSample = profile.samples[sampleIdx + 1]; + const timestamp = nextSample ? nextSample.start_timestamp : endTs; + + if (timestamp > endTs) { + continue; + } + const commonAttributes = { + start_timestamp: sampleTs, + timestamp, + trace_id: trace.trace_id, + status: "ok", + tags: { source: "profile" }, + data: { + "thread.id": sample.thread_id, + "thread.name": profile.thread_metadata?.[sample.thread_id as keyof typeof profile.thread_metadata]?.name, + }, + }; + const identity = [trace.trace_id, profile.timestamp, parent_span_id, startTs, endTs, sampleIdx, sample.thread_id]; + const sampleSpan: Span = { + span_id: profileSpanId([...identity, "thread"]), + parent_span_id, + ...commonAttributes, + op: "Thread", + description: + profile.thread_metadata?.[sample.thread_id as keyof typeof profile.thread_metadata]?.name || + `Thread ${sample.thread_id}`, + data: { + thread_id: sample.thread_id, + }, + }; + let currentSpan = sampleSpan; + const currentStack = profile.stacks[sample.stack_id]; + const lastSentryFrameIdx = sentryFrameFilter ? currentStack.findLastIndex(sentryFrameFilter, profile.frames) : 0; + for (let frameIdxIdx = lastSentryFrameIdx + 1; frameIdxIdx < currentStack.length; frameIdxIdx++) { + const frame = profile.frames[currentStack[frameIdxIdx]]; + // XXX: We may wanna skip frames that doesn't have `in_app` set to true + // that said it's better to have this as a dynamic filter + const [description, op] = getFunctionNameFromFrame(frame).split("@"); + const spanFromFrame: Span = { + span_id: profileSpanId([...identity, "frame", frameIdxIdx]), + parent_span_id: currentSpan.span_id, + ...commonAttributes, + op, + description, + data: { + ...frame, + }, + }; + currentSpan.children = [spanFromFrame]; + currentSpan = spanFromFrame; + } + fillerSpans.push(sampleSpan); + } + + if (!fillerSpans.length) { + return []; + } + const consolidated = consolidateSpans(trace, fillerSpans); + // Remove the extra layer of nesting if there is only one span which should be the "Thread" span + return (consolidated.length === 1 ? consolidated[0].children || [] : consolidated).filter( + span => span.timestamp - span.start_timestamp > 0 && span.timestamp <= endTs, + ); +} + +/** + * Modifies the spanTree in place recursively by adding spans from the + * profile data where there are gaps in the trace data. + * @param spanTree Span[] The tree of spans to graft profile spans into + */ +export function graftProfileSpans( + trace: Trace, + profile: SentryProfileWithTraceMeta, + spanTree: Span[] = trace.spanTree, + parent: Span | Trace = trace, +) { + log(`Grafting profile spans into trace ${trace.trace_id}`); + if (trace.profileGrafted) { + log(`Trace already has profile grafted ${trace.trace_id}`); + return; + } + + let idx = -1; + while (idx < spanTree.length) { + const span = spanTree[idx] as Span | undefined; + if (span?.tags?.source === "profile") { + idx += 1; + continue; + } + const nextSpan = spanTree[idx + 1]; + if (nextSpan?.tags?.source === "profile") { + idx += 1; + continue; + } + + const startTs = span ? span.timestamp : parent.start_timestamp; + const endTs = nextSpan ? nextSpan.start_timestamp : parent.timestamp; + const threadIds = new Set([span?.data?.threadId, nextSpan?.data?.threadId, parent?.data?.threadId]); + threadIds.delete(undefined); + if (endTs - startTs > 0) { + const fillers = getSpansFromProfile(trace, profile, parent.span_id, startTs, endTs, threadIds as Set); + if (fillers.length) { + spanTree.splice(idx + 1, 0, ...fillers); + idx += fillers.length; + } + } + if (span) { + span.children ??= []; + graftProfileSpans(trace, profile, span.children, span); + } + idx += 1; + } + // Only mark as grafted at the top level to avoid early quitting during + // recursive calls above for child spans + trace.profileGrafted = trace.spanTree === spanTree; +} diff --git a/packages/spotlight/src/telemetry/index.ts b/packages/spotlight/src/telemetry/index.ts new file mode 100644 index 000000000..b91306258 --- /dev/null +++ b/packages/spotlight/src/telemetry/index.ts @@ -0,0 +1,340 @@ +import type { Envelope } from "@sentry/core"; +import { createEngine } from "./internal/engine.ts"; +import { normalizeEnvelope, supportsEnvelope, validateEnvelope } from "./internal/envelopes.ts"; +import { readonlyCopy } from "./internal/readonly.ts"; +import type { + Diagnostic, + EventFilter, + IngestResult, + Telemetry, + TelemetryOptions, + TelemetrySnapshot, + TraceFilter, +} from "./public.ts"; +import { + getAITraces, + getProfileFunctionSummaries, + getQuerySummaries, + getResourceSummaries, + getSpans, + getWebVitals, +} from "./selectors/index.ts"; +import { normalizeSpanItem } from "./store/utils/spanProcessor.ts"; +import type { SentryErrorEvent, SentryEvent, Trace } from "./types.ts"; +import { isErrorEvent } from "./utils/sentry.ts"; +import { generateUuidv4 } from "./uuid.ts"; + +type RetainedEnvelope = { id: string; envelope: Envelope; receivedAtMs: number }; + +/** Portable, independently owned telemetry with immutable committed reads. */ +export function createTelemetry(options: TelemetryOptions = {}): Telemetry { + const maxEnvelopes = options.retention?.maxEnvelopes ?? 500; + const maxAgeMs = options.retention?.maxAgeMs; + if (!Number.isSafeInteger(maxEnvelopes) || maxEnvelopes < 1) + throw new RangeError("retention.maxEnvelopes must be a positive safe integer"); + if (maxAgeMs !== undefined && (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0)) + throw new RangeError("retention.maxAgeMs must be a positive finite number"); + let generation = 0; + let disposed = false; + let revision = 0; + let queue: Promise = Promise.resolve(); + let journal: RetainedEnvelope[] = []; + let timer: ReturnType | undefined; + const identities = new WeakMap(); + const listeners = new Set<(snapshot: TelemetrySnapshot) => void>(); + let enrichmentCache = new Map(); + let currentDiagnostics: Diagnostic[] = []; + function diagnostic(item: Diagnostic) { + currentDiagnostics.push(item); + try { + options.onDiagnostic?.(item); + } catch { + /* Diagnostic observers cannot break capture. */ + } + } + function makeEngine() { + const cache = enrichmentCache; + const epoch = generation; + let currentEnvelopeId = ""; + const created = createEngine({ + enrichStacktrace: async event => { + const envelopeId = currentEnvelopeId; + const cached = cache.get(event.event_id); + if (cached?.envelopeId === envelopeId) { + event.exception = structuredClone(cached.exception); + return; + } + try { + await options.enrichStacktrace?.(event); + } catch { + if (active(epoch) && created.runtime.generation === 0) + diagnostic({ + code: "enrichment_failed", + message: "Stacktrace enrichment failed; captured telemetry was retained.", + }); + } + if (active(epoch) && created.runtime.generation === 0) + cache.set(event.event_id, { envelopeId, exception: structuredClone(event.exception) }); + }, + }); + return { + ...created, + ingestEnvelope(envelope: Envelope, receivedAtMs: number) { + currentEnvelopeId = String(envelope[0].__spotlight_envelope_id); + created.runtime.receivedAtMs = receivedAtMs; + return created.store.getState().ingestEnvelope(envelope); + }, + }; + } + function invalid(code: string, message: string): IngestResult { + const item = { code, message }; + try { + options.onDiagnostic?.(item); + } catch { + /* Diagnostic observers do not break ingestion. */ + } + return readonlyCopy({ status: "invalid" as const, diagnostics: [item] }); + } + let engine = makeEngine(); + function stateSnapshot(): TelemetrySnapshot { + const state = engine.store.getState(); + return readonlyCopy({ + revision, + envelopes: state.getEnvelopes(), + events: state.getEvents(), + traces: state.getTraces(), + logs: state.getLogs(), + sdks: state.getSdks(), + profiles: state.profilesByTraceId, + }); + } + let snapshot = stateSnapshot(); + function publish() { + revision++; + snapshot = stateSnapshot(); + const committed = snapshot; + for (const listener of listeners) { + try { + listener(committed); + } catch { + diagnostic({ code: "subscriber_failed", message: "A Telemetry subscription callback failed." }); + } + if (snapshot !== committed) break; + } + } + function clearTimer() { + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + } + function scheduleExpiry() { + clearTimer(); + if (disposed || maxAgeMs === undefined || !journal.length) return; + const delay = Math.max(1, Math.min(2147483647, journal[0].receivedAtMs + maxAgeMs - Date.now())); + timer = setTimeout(() => { + timer = undefined; + void client.prune(); + }, delay); + if (typeof timer === "object" && timer && "unref" in timer) timer.unref(); + } + function enqueue(task: (epoch: number) => Promise): Promise { + const epoch = generation; + const next = queue.then(() => task(epoch)); + queue = next.catch(() => {}); + return next; + } + function ignored(reason: string, envelopeId?: string): IngestResult { + return readonlyCopy({ status: "ignored" as const, reason, ...(envelopeId ? { envelopeId } : {}), diagnostics: [] }); + } + function active(epoch: number) { + return !disposed && epoch === generation; + } + async function rebuild(retained: RetainedEnvelope[], epoch: number) { + const replacement = makeEngine(); + for (const record of retained) { + await replacement.ingestEnvelope(record.envelope, record.receivedAtMs); + if (!active(epoch)) return; + } + engine = replacement; + const eventIds = engine.store.getState().eventsById; + for (const key of enrichmentCache.keys()) if (!eventIds.has(key)) enrichmentCache.delete(key); + } + async function enforceRetention(epoch: number): Promise { + const cutoff = maxAgeMs === undefined ? Number.NEGATIVE_INFINITY : Date.now() - maxAgeMs; + const retained = journal.filter(record => record.receivedAtMs > cutoff).slice(-maxEnvelopes); + if (retained.length === journal.length) return false; + await rebuild(retained, epoch); + if (!active(epoch)) return false; + journal = retained; + return true; + } + function filterEvents(filter?: EventFilter) { + return snapshot.events.filter( + event => + (filter?.traceId === undefined || event.contexts?.trace?.trace_id === filter.traceId) && + (filter?.spanId === undefined || event.contexts?.trace?.span_id === filter.spanId), + ); + } + function filteredTraces(filter?: TraceFilter) { + return snapshot.traces.filter(trace => filter?.traceId === undefined || trace.trace_id === filter.traceId); + } + const client: Telemetry = { + ingestEnvelope: input => { + if (disposed) return Promise.resolve(ignored("disposed")); + if (!validateEnvelope(input)) + return Promise.resolve( + invalid("invalid_envelope", "Expected a parsed envelope [headers, [[itemHeaders, payload], ...]]."), + ); + const supported = supportsEnvelope(input); + const inputDiagnostics: Diagnostic[] = []; + let validSpans = 0; + let invalidSpans = 0; + let otherItems = false; + for (const [header, payload] of input[1]) { + if (header.type !== "span") { + otherItems = true; + continue; + } + const data = payload as Record; + if ("items" in data && data.version !== 2) continue; + const count = Array.isArray(data.items) ? data.items.length : 1; + const valid = normalizeSpanItem(data, { name: "unknown", version: "unknown", lastSeen: 0 }).length; + validSpans += valid; + invalidSpans += count - valid; + } + if (invalidSpans > 0) { + if (validSpans === 0 && !otherItems) + return Promise.resolve(invalid("invalid_span", "Span items are missing valid IDs or timestamps.")); + inputDiagnostics.push({ + code: "invalid_span", + message: "Malformed span items were skipped; valid telemetry was retained.", + }); + } + let id: string; + let envelope: Envelope; + try { + id = String(input[0].__spotlight_envelope_id ?? identities.get(input) ?? generateUuidv4()); + identities.set(input, id); + envelope = normalizeEnvelope(input); + envelope[0].__spotlight_envelope_id = id; + } catch { + return Promise.resolve(invalid("invalid_envelope", "Envelope data must be cloneable wire data.")); + } + const receivedAtMs = Date.now(); + return enqueue(async epoch => { + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + currentDiagnostics = []; + for (const item of inputDiagnostics) diagnostic(item); + const evicted = await enforceRetention(epoch); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (journal.some(record => record.id === id)) { + if (evicted) publish(); + scheduleExpiry(); + return ignored("duplicate", id); + } + const working = engine; + try { + await working.ingestEnvelope(envelope, receivedAtMs); + } catch { + working.store.getState().resetData(); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + await rebuild(journal, epoch); + if (active(epoch)) { + if (evicted) publish(); + scheduleExpiry(); + } + return invalid("invalid_envelope", "Envelope telemetry could not be processed."); + } + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + journal.push({ id, envelope, receivedAtMs }); + await enforceRetention(epoch); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + publish(); + scheduleExpiry(); + return supported + ? readonlyCopy({ status: "processed" as const, envelopeId: id, diagnostics: currentDiagnostics }) + : readonlyCopy({ + status: "ignored" as const, + reason: "unsupported_items", + envelopeId: id, + diagnostics: currentDiagnostics, + }); + }); + }, + getSnapshot: () => snapshot, + getEnvelope: id => snapshot.envelopes.find(envelope => String(envelope[0].__spotlight_envelope_id) === id), + getEnvelopes: () => snapshot.envelopes, + getEvent: id => snapshot.events.find(event => event.event_id === id), + getEvents: filter => (filter ? Object.freeze(filterEvents(filter)) : snapshot.events), + getErrors: filter => + Object.freeze(filterEvents(filter).filter(event => isErrorEvent(event as SentryEvent))) as ReturnType< + Telemetry["getErrors"] + >, + getTrace: id => snapshot.traces.find(trace => trace.trace_id === id), + getTraces: filter => (filter ? Object.freeze(filteredTraces(filter)) : snapshot.traces), + getSpans: filter => Object.freeze(getSpans(filteredTraces(filter) as Trace[])) as ReturnType, + getLog: id => snapshot.logs.find(log => log.id === id), + getLogs: filter => + filter?.traceId === undefined + ? snapshot.logs + : Object.freeze(snapshot.logs.filter(log => log.trace_id === filter.traceId)), + getProfile: id => snapshot.profiles.get(id), + getProfileFunctionSummaries: filter => + readonlyCopy( + getProfileFunctionSummaries( + snapshot.profiles as unknown as Parameters[0], + filter, + ), + ), + getQuerySummaries: filter => + readonlyCopy( + getQuerySummaries(getSpans(filteredTraces(filter) as Trace[]), { description: filter?.description }), + ), + getResourceSummaries: filter => readonlyCopy(getResourceSummaries(getSpans(filteredTraces(filter) as Trace[]))), + getAITraces: filter => readonlyCopy(getAITraces(filteredTraces(filter) as Trace[])), + getWebVitals: () => readonlyCopy(getWebVitals(snapshot.events as SentryEvent[])), + getSdks: () => snapshot.sdks, + subscribe: listener => { + if (disposed) return () => {}; + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, + reset: () => { + if (disposed) return; + generation++; + clearTimer(); + engine.store.getState().resetData(); + journal = []; + enrichmentCache = new Map(); + engine = makeEngine(); + queue = Promise.resolve(); + publish(); + }, + prune: () => + enqueue(async epoch => { + if (!active(epoch)) return; + if (await enforceRetention(epoch)) { + if (active(epoch)) publish(); + } + scheduleExpiry(); + }), + dispose: () => { + if (disposed) return; + generation++; + disposed = true; + clearTimer(); + engine.store.getState().resetData(); + journal = []; + enrichmentCache.clear(); + listeners.clear(); + engine = makeEngine(); + revision++; + snapshot = stateSnapshot(); + queue = Promise.resolve(); + }, + }; + return client; +} + +export type { Telemetry, TelemetryOptions, TelemetrySnapshot, IngestResult, Diagnostic } from "./public.ts"; diff --git a/packages/spotlight/src/telemetry/internal/engine.ts b/packages/spotlight/src/telemetry/internal/engine.ts new file mode 100644 index 000000000..26a978b4c --- /dev/null +++ b/packages/spotlight/src/telemetry/internal/engine.ts @@ -0,0 +1,27 @@ +import { createStore } from "zustand/vanilla"; +import { createEnvelopesSlice } from "../store/slices/envelopesSlice.ts"; +import { createEventsSlice } from "../store/slices/eventsSlice.ts"; +import { createLogsSlice } from "../store/slices/logsSlice.ts"; +import { createProfilesSlice } from "../store/slices/profilesSlice.ts"; +import { createSDKsSlice } from "../store/slices/sdksSlice.ts"; +import { createSharedSlice } from "../store/slices/sharedSlice.ts"; +import { createSubscriptionsSlice } from "../store/slices/subscriptionsSlice.ts"; +import { createTracesSlice } from "../store/slices/tracesSlice.ts"; +import type { SentryStore } from "../store/types.ts"; + +import type { EngineOptions, ProcessingRuntime } from "./types.ts"; + +export function createEngine(options: EngineOptions = {}) { + const runtime: ProcessingRuntime = { generation: 0, receivedAtMs: Date.now() }; + const store = createStore()((...args) => ({ + ...createEventsSlice(options, runtime)(...args), + ...createTracesSlice(...args), + ...createProfilesSlice(...args), + ...createSubscriptionsSlice(...args), + ...createEnvelopesSlice(runtime)(...args), + ...createSDKsSlice(...args), + ...createLogsSlice(...args), + ...createSharedSlice(runtime)(...args), + })); + return { store, runtime }; +} diff --git a/packages/spotlight/src/telemetry/internal/envelopes.ts b/packages/spotlight/src/telemetry/internal/envelopes.ts new file mode 100644 index 000000000..2042e2f58 --- /dev/null +++ b/packages/spotlight/src/telemetry/internal/envelopes.ts @@ -0,0 +1,61 @@ +import type { Envelope } from "@sentry/core"; +import { SUPPORTED_EVENT_TYPES } from "../constants/sentry.ts"; + +export function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} +export function validateEnvelope(input: unknown): input is Envelope { + if (!Array.isArray(input) || input.length !== 2 || !isRecord(input[0]) || !Array.isArray(input[1])) return false; + return input[1].every( + item => + Array.isArray(item) && + item.length === 2 && + isRecord(item[0]) && + typeof item[0].type === "string" && + ((isRecord(item[1]) && !(item[1] instanceof Uint8Array)) || + (!SUPPORTED_EVENT_TYPES.has(item[0].type) && + item[0].type !== "span" && + (typeof item[1] === "string" || item[1] instanceof Uint8Array))), + ); +} + +const TEXT_CONTENT_TYPES = new Set([ + "text/plain", + "text/css", + "text/csv", + "text/html", + "text/javascript", + "text/json", + "text/x-json", + "application/json", + "application/ld+json", +]); + +/** Match the wire decoder's attachment representation without requiring Node's Buffer. */ +export function normalizeEnvelope(input: Envelope): Envelope { + const envelope = structuredClone(input); + for (const item of envelope[1]) { + const [header, payload] = item; + if (typeof payload !== "string" && !(payload instanceof Uint8Array)) continue; + const bytes = typeof payload === "string" ? new TextEncoder().encode(payload) : payload; + let data: string; + if (TEXT_CONTENT_TYPES.has(header.content_type as string)) { + data = new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + } else { + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + data = btoa(binary); + } + item[1] = { data } as (typeof item)[1]; + } + return envelope; +} +export function supportsEnvelope(input: Envelope): boolean { + return input[1].some(([header, payload]) => { + if (header.type !== "span") return SUPPORTED_EVENT_TYPES.has(header.type); + const span = payload as Record; + return !("items" in span) || (span.version === 2 && Array.isArray(span.items)); + }); +} diff --git a/packages/spotlight/src/telemetry/internal/readonly.ts b/packages/spotlight/src/telemetry/internal/readonly.ts new file mode 100644 index 000000000..5128718d3 --- /dev/null +++ b/packages/spotlight/src/telemetry/internal/readonly.ts @@ -0,0 +1,59 @@ +import type { DeepReadonly } from "../public.ts"; + +/** Copies the graph before freezing it; Map/Set mutation methods never expose the backing collection. */ +export function readonlyCopy(input: T): DeepReadonly { + const copies = new WeakMap(); + function copy(value: unknown): unknown { + if (!value || typeof value !== "object") return value; + if (copies.has(value)) return copies.get(value); + if (value instanceof Map || value instanceof Set) { + const target = value instanceof Map ? new Map() : new Set(); + const proxy = new Proxy(target, { + get(collection, key) { + if (["set", "add", "delete", "clear"].includes(String(key))) + return () => { + throw new TypeError("Telemetry snapshots are read-only"); + }; + if (key === "valueOf") return () => proxy; + if (key === "forEach") + return (callback: (value: unknown, key: unknown, collection: unknown) => void, thisArg?: unknown) => { + collection.forEach((item, itemKey) => callback.call(thisArg, item, itemKey, proxy)); + }; + const member = Reflect.get(collection, key, collection); + return typeof member === "function" && + (["get", "has", "entries", "keys", "values"].includes(String(key)) || key === Symbol.iterator) + ? member.bind(collection) + : member; + }, + setPrototypeOf() { + throw new TypeError("Telemetry snapshots are read-only"); + }, + set() { + throw new TypeError("Telemetry snapshots are read-only"); + }, + defineProperty() { + throw new TypeError("Telemetry snapshots are read-only"); + }, + deleteProperty() { + throw new TypeError("Telemetry snapshots are read-only"); + }, + }); + copies.set(value, proxy); + if (value instanceof Map && target instanceof Map) + for (const [key, item] of value) target.set(copy(key), copy(item)); + else if (value instanceof Set && target instanceof Set) for (const item of value) target.add(copy(item)); + return proxy; + } + const result: Record | unknown[] = Array.isArray(value) ? [] : {}; + copies.set(value, result); + for (const key of Object.keys(value)) + Object.defineProperty(result, key, { + value: copy((value as Record)[key]), + enumerable: true, + configurable: true, + writable: true, + }); + return Object.freeze(result); + } + return copy(input) as DeepReadonly; +} diff --git a/packages/spotlight/src/telemetry/internal/types.ts b/packages/spotlight/src/telemetry/internal/types.ts new file mode 100644 index 000000000..64502db52 --- /dev/null +++ b/packages/spotlight/src/telemetry/internal/types.ts @@ -0,0 +1,3 @@ +import type { SentryErrorEvent } from "../types.ts"; +export type EngineOptions = { enrichStacktrace?: (event: SentryErrorEvent) => Promise }; +export type ProcessingRuntime = { generation: number; receivedAtMs: number }; diff --git a/packages/spotlight/src/telemetry/logger.ts b/packages/spotlight/src/telemetry/logger.ts new file mode 100644 index 000000000..23669ea52 --- /dev/null +++ b/packages/spotlight/src/telemetry/logger.ts @@ -0,0 +1,23 @@ +export const TELEMETRY_PREFIX = "🔎 [Telemetry]"; + +const noop = (..._args: unknown[]) => {}; // eslint-disable-line @typescript-eslint/no-unused-vars +let _log = noop; +let _warn = noop; + +export function activateLogger() { + _log = (...args: unknown[]) => console.log(TELEMETRY_PREFIX, ...args); + _warn = (...args: unknown[]) => console.warn(TELEMETRY_PREFIX, ...args); +} + +export function deactivateLogger() { + _log = noop; + _warn = noop; +} + +export function log(...args: unknown[]) { + _log(...args); +} + +export function warn(...args: unknown[]) { + _warn(...args); +} diff --git a/packages/spotlight/src/telemetry/parity.spec.ts b/packages/spotlight/src/telemetry/parity.spec.ts new file mode 100644 index 000000000..0940d8e79 --- /dev/null +++ b/packages/spotlight/src/telemetry/parity.spec.ts @@ -0,0 +1,88 @@ +// @vitest-environment node +import { readFileSync } from "node:fs"; +import type { Envelope } from "@sentry/core"; +import { expect, test } from "vitest"; +import { createTelemetry } from "../sdk/core.ts"; +import { createNodeTelemetry } from "../sdk/node.ts"; +import useSentryStore from "../ui/telemetry/store/store"; +import { getAITraces, getQuerySummaries, getSpans } from "./selectors/index.ts"; + +test("the UI store, browser processor, and Node SDK derive identical traces and queries", async () => { + const spotlight = createNodeTelemetry(); + const browser = createTelemetry(); + const result = await spotlight.ingest({ + body: readFileSync(new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url)), + contentType: "application/x-sentry-envelope", + }); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + const envelope = spotlight.getEnvelope(result.envelopeId)!; + const original = JSON.stringify(envelope); + useSentryStore.getState().resetData(); + await useSentryStore.getState().ingestEnvelope(envelope); + await browser.ingestEnvelope(envelope); + const uiTraces = useSentryStore.getState().getTraces(); + expect(uiTraces).toEqual(spotlight.getTraces()); + expect(browser.getTraces()).toEqual(spotlight.getTraces()); + expect(getQuerySummaries(getSpans(uiTraces))).toEqual(spotlight.getQuerySummaries()); + expect(browser.getQuerySummaries()).toEqual(spotlight.getQuerySummaries()); + expect(JSON.stringify(envelope)).toBe(original); + useSentryStore.getState().resetData(); +}); + +test("the UI, browser processor, and Node SDK share log, profile, and AI processing", async () => { + const spotlight = createNodeTelemetry(); + const browser = createTelemetry(); + useSentryStore.getState().resetData(); + for (const file of ["continuous_profiling/transaction.txt", "continuous_profiling/profile_chunk_1.txt"]) { + const result = await spotlight.ingest({ + body: readFileSync(new URL(`../../_fixtures/${file}`, import.meta.url)), + contentType: "application/x-sentry-envelope", + }); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + const input = spotlight.getEnvelope(result.envelopeId)!; + await browser.ingestEnvelope(input); + await useSentryStore.getState().ingestEnvelope(input); + } + const traceId = "f1e2d3c4b5a6978012345678901234ab"; + const input = [ + { sdk: { name: "sentry.javascript.node", version: "11.9.0" } }, + [ + [{ type: "log" }, { items: [{ id: "log-1", body: "hello", timestamp: 1, trace_id: traceId, attributes: {} }] }], + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "ai-trace", + span_id: "ai", + name: "chat", + start_timestamp: 1, + end_timestamp: 2, + attributes: { "sentry.op": { value: "gen_ai.chat" }, "gen_ai.request.model": { value: "model" } }, + }, + ], + }, + ], + ], + ] as unknown as Envelope; + await spotlight.ingestEnvelope(input); + await browser.ingestEnvelope(input); + await useSentryStore.getState().ingestEnvelope(input); + const ui = useSentryStore.getState(); + expect(ui.getLogsByTraceId(traceId)).toEqual(spotlight.getLogs({ traceId })); + expect(browser.getLogs()).toEqual(spotlight.getLogs()); + expect(ui.getProfileByTraceId(traceId)).toEqual(spotlight.getProfile(traceId)); + expect(browser.getProfile(traceId)).toEqual(spotlight.getProfile(traceId)); + expect(ui.getAggregateCallData()).toEqual( + spotlight.getProfileFunctionSummaries().map(summary => ({ + name: summary.name, + totalTime: summary.totalDurationMs, + samples: summary.sampleCount, + traceIds: summary.traceIds, + })), + ); + expect(getAITraces(ui.getTraces())).toEqual(spotlight.getAITraces()); + expect(browser.getAITraces()).toEqual(spotlight.getAITraces()); + useSentryStore.getState().resetData(); +}); diff --git a/packages/spotlight/src/telemetry/public.ts b/packages/spotlight/src/telemetry/public.ts new file mode 100644 index 000000000..f4a8a3767 --- /dev/null +++ b/packages/spotlight/src/telemetry/public.ts @@ -0,0 +1,100 @@ +import type { Envelope } from "@sentry/core"; +import type { + ProfileFunctionSummary as InternalProfileFunctionSummary, + QuerySummary as InternalQuerySummary, + ResourceSummary as InternalResourceSummary, +} from "./selectors/index.ts"; + +import type { + AITrace as InternalAITrace, + Sdk, + SentryErrorEvent, + SentryEvent, + SentryEventWithPerformanceData, + SentryLogEventItem, + SentryProfileWithTraceMeta, + Span, + Trace, +} from "./types.ts"; + +export type DeepReadonly = T extends (...args: never[]) => unknown + ? T + : T extends ReadonlyMap + ? ReadonlyMap, DeepReadonly> + : T extends ReadonlySet + ? ReadonlySet> + : T extends readonly unknown[] + ? { readonly [K in keyof T]: DeepReadonly } + : T extends object + ? { readonly [K in keyof T]: DeepReadonly } + : T; + +/** A parsed Sentry wire envelope; input timestamps retain their wire units. */ +export type RawEnvelope = Envelope; +export type ProcessedEvent = DeepReadonly; +export type ProcessedSpan = DeepReadonly; +export type ProcessedTrace = DeepReadonly; +export type ProcessedLog = DeepReadonly; +export type ProcessedProfile = DeepReadonly; +export type AITrace = DeepReadonly; +export type QuerySummary = DeepReadonly; +export type ResourceSummary = DeepReadonly; +export type ProfileFunctionSummary = DeepReadonly; +export type Diagnostic = { readonly code: string; readonly message: string }; +export type IngestResult = + | { readonly status: "processed"; readonly envelopeId: string; readonly diagnostics: readonly Diagnostic[] } + | { + readonly status: "ignored"; + readonly reason: string; + readonly envelopeId?: string; + readonly diagnostics: readonly Diagnostic[]; + } + | { readonly status: "invalid"; readonly diagnostics: readonly Diagnostic[] }; +export type TraceFilter = { traceId?: string }; +export type EventFilter = TraceFilter & { spanId?: string }; +export type DatabaseQueryFilter = TraceFilter & { description?: string }; +export type RetentionOptions = { + /** Maximum retained envelopes, including all telemetry derived from them. Default: 500. */ + maxEnvelopes?: number; + /** Optional maximum age since receipt. Expiration runs automatically; dispose releases the timer. */ + maxAgeMs?: number; +}; +export type TelemetryOptions = { + retention?: RetentionOptions; + enrichStacktrace?: (event: SentryErrorEvent) => Promise; + onDiagnostic?: (diagnostic: Diagnostic) => void; +}; +export type TelemetrySnapshot = { + readonly revision: number; + readonly envelopes: readonly DeepReadonly[]; + readonly events: readonly ProcessedEvent[]; + readonly traces: readonly ProcessedTrace[]; + readonly logs: readonly ProcessedLog[]; + readonly sdks: readonly DeepReadonly[]; + readonly profiles: ReadonlyMap; +}; +export type Telemetry = { + ingestEnvelope: (input: unknown) => Promise; + getSnapshot: () => TelemetrySnapshot; + getEnvelope: (id: string) => DeepReadonly | undefined; + getEnvelopes: () => readonly DeepReadonly[]; + getEvent: (id: string) => ProcessedEvent | undefined; + getEvents: (filter?: EventFilter) => readonly ProcessedEvent[]; + getErrors: (filter?: EventFilter) => readonly DeepReadonly[]; + getTrace: (id: string) => ProcessedTrace | undefined; + getTraces: (filter?: TraceFilter) => readonly ProcessedTrace[]; + getSpans: (filter?: TraceFilter) => readonly ProcessedSpan[]; + getLog: (id: string) => ProcessedLog | undefined; + getLogs: (filter?: TraceFilter) => readonly ProcessedLog[]; + getProfile: (traceId: string) => ProcessedProfile | undefined; + getProfileFunctionSummaries: (filter?: TraceFilter) => readonly ProfileFunctionSummary[]; + getQuerySummaries: (filter?: DatabaseQueryFilter) => readonly QuerySummary[]; + getResourceSummaries: (filter?: TraceFilter) => readonly ResourceSummary[]; + getAITraces: (filter?: TraceFilter) => readonly AITrace[]; + getWebVitals: () => readonly DeepReadonly[]; + getSdks: () => readonly DeepReadonly[]; + subscribe: (listener: (snapshot: TelemetrySnapshot) => void) => () => void; + reset: () => void; + prune: () => Promise; + dispose: () => void; +}; diff --git a/packages/spotlight/src/telemetry/queries.ts b/packages/spotlight/src/telemetry/queries.ts new file mode 100644 index 000000000..5684ae41c --- /dev/null +++ b/packages/spotlight/src/telemetry/queries.ts @@ -0,0 +1,2 @@ +// Internal compatibility barrel. Prefer importing from the individual selector domain. +export * from "./selectors/index.ts"; diff --git a/packages/spotlight/src/telemetry/selectors/ai.ts b/packages/spotlight/src/telemetry/selectors/ai.ts new file mode 100644 index 000000000..f0e72fc52 --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/ai.ts @@ -0,0 +1,13 @@ +import { createAITraceFromSpan, extractAllAIRootSpans } from "../ai/aiLibraries.ts"; +import type { AITrace, Span, Trace } from "../types.ts"; + +export function getAIRootSpans(traces: Iterable): Span[] { + return Array.from(traces).flatMap(trace => extractAllAIRootSpans(trace.spanTree).map(({ span }) => span)); +} + +export function getAITraces(traces: Iterable): AITrace[] { + return getAIRootSpans(traces).flatMap(span => { + const result = createAITraceFromSpan(span); + return result ? [result] : []; + }); +} diff --git a/packages/spotlight/src/telemetry/selectors/database.ts b/packages/spotlight/src/telemetry/selectors/database.ts new file mode 100644 index 000000000..5007d4214 --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/database.ts @@ -0,0 +1,15 @@ +import type { Span } from "../types.ts"; +import { type SpanFilter, type SpanSummary, summarizeSpans } from "./spans.ts"; + +export const DB_SPAN_REGEX = /^db(?:\.[A-Za-z]+)*$/; +export type QueryFilter = SpanFilter & { description?: string }; +export type QuerySummary = SpanSummary; + +/** Inputs and durations use normalized milliseconds. Presentation sorting belongs to callers. */ +export function getQuerySummaries(spans: readonly Span[], filter: QueryFilter = {}): QuerySummary[] { + return summarizeSpans( + spans.filter( + span => DB_SPAN_REGEX.test(span.op ?? "") && (filter.traceId === undefined || span.trace_id === filter.traceId), + ), + ).filter(query => filter.description === undefined || query.description === filter.description.trim()); +} diff --git a/packages/spotlight/src/telemetry/selectors/index.ts b/packages/spotlight/src/telemetry/selectors/index.ts new file mode 100644 index 000000000..8e5913480 --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/index.ts @@ -0,0 +1,6 @@ +export { getAIRootSpans, getAITraces } from "./ai.ts"; +export { DB_SPAN_REGEX, getQuerySummaries, type QueryFilter, type QuerySummary } from "./database.ts"; +export { getProfileFunctionSummaries, type ProfileFunctionSummary } from "./profiles.ts"; +export { getResourceSummaries, type ResourceSummary } from "./resources.ts"; +export { getSpans, type SpanFilter } from "./spans.ts"; +export { getWebVitals } from "./webVitals.ts"; diff --git a/packages/spotlight/src/telemetry/selectors/profiles.ts b/packages/spotlight/src/telemetry/selectors/profiles.ts new file mode 100644 index 000000000..162a81d37 --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/profiles.ts @@ -0,0 +1,43 @@ +import type { SentryProcessedProfile, TraceId } from "../types.ts"; +import { getFunctionNameFromFrame } from "../utils/profileUtils.ts"; +import type { SpanFilter } from "./spans.ts"; + +export type ProfileFunctionSummary = { + name: string; + totalDurationMs: number; + sampleCount: number; + traceIds: Set; +}; + +/** Inclusive frame durations in milliseconds, using each sample's interval to the next sample. */ +export function getProfileFunctionSummaries( + profilesByTraceId: ReadonlyMap, + filter: SpanFilter = {}, +): ProfileFunctionSummary[] { + const aggregateCalls = new Map(); + for (const [traceId, profile] of profilesByTraceId) { + if (filter.traceId !== undefined && traceId !== filter.traceId) continue; + // The final sample has no known duration and is not included. + for (let sampleIdx = 0; sampleIdx < profile.samples.length - 1; sampleIdx++) { + const sample = profile.samples[sampleIdx]; + const duration = profile.samples[sampleIdx + 1].start_timestamp - sample.start_timestamp; + for (const frameIdx of profile.stacks[sample.stack_id]) { + const name = getFunctionNameFromFrame(profile.frames[frameIdx]); + const callData = aggregateCalls.get(name); + if (callData) { + callData.totalDurationMs += duration; + callData.sampleCount += 1; + callData.traceIds.add(traceId); + } else { + aggregateCalls.set(name, { + name, + totalDurationMs: duration, + sampleCount: 1, + traceIds: new Set([traceId]), + }); + } + } + } + } + return Array.from(aggregateCalls.values()); +} diff --git a/packages/spotlight/src/telemetry/selectors/resources.ts b/packages/spotlight/src/telemetry/selectors/resources.ts new file mode 100644 index 000000000..0035dcbbf --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/resources.ts @@ -0,0 +1,15 @@ +import type { Span } from "../types.ts"; +import { type SpanSummary, summarizeSpans } from "./spans.ts"; + +export type ResourceSummary = SpanSummary & { averageEncodedSizeBytes: number }; + +export function getResourceSummaries(spans: readonly Span[]): ResourceSummary[] { + return summarizeSpans(spans.filter(span => /resource\.[A-Za-z]+/.test(span.op ?? ""))).map(group => ({ + ...group, + averageEncodedSizeBytes: + group.spans.reduce((sum, span) => { + const size = span.data?.["http.response_content_length"]; + return sum + (typeof size === "number" ? size : 0); + }, 0) / group.count, + })); +} diff --git a/packages/spotlight/src/telemetry/selectors/selectors.test.ts b/packages/spotlight/src/telemetry/selectors/selectors.test.ts new file mode 100644 index 000000000..88646304b --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/selectors.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; +import type { SentryProcessedProfile, Span } from "../types.ts"; +import { getQuerySummaries } from "./database.ts"; +import { getProfileFunctionSummaries } from "./profiles.ts"; +import { getResourceSummaries } from "./resources.ts"; + +function span(overrides: Partial = {}): Span { + return { + span_id: "span-id", + trace_id: "trace-a", + op: "db.sql.query", + description: " SELECT 1 ", + start_timestamp: 1000, + timestamp: 1010, + ...overrides, + }; +} + +function profile(): SentryProcessedProfile { + return { + frames: [ + { function: "main", module: "app" }, + { function: "query", module: "app" }, + { function: "final", module: "app" }, + ], + stacks: [[0, 1], [0], [2]], + samples: [ + { start_timestamp: 1000, stack_id: 0, thread_id: "main" }, + { start_timestamp: 1010, stack_id: 1, thread_id: "main" }, + { start_timestamp: 1040, stack_id: 2, thread_id: "main" }, + ], + }; +} + +describe("domain summaries", () => { + it("groups database spans using normalized millisecond fields and excludes invalid inputs", () => { + const first = span(); + const second = span({ span_id: "second", description: "SELECT 1", timestamp: 1030 }); + const result = getQuerySummaries([ + first, + span({ op: "http.client" }), + second, + span({ description: " " }), + span({ timestamp: Number.NaN }), + span({ start_timestamp: Number.POSITIVE_INFINITY }), + ]); + + expect(result).toEqual([ + { + description: "SELECT 1", + count: 2, + totalDurationMs: 40, + averageDurationMs: 20, + spans: [first, second], + }, + ]); + expect(first.description).toBe(" SELECT 1 "); + expect(getQuerySummaries([first], { description: " SELECT 1 " })).toHaveLength(1); + expect(getQuerySummaries([first], { description: "SELECT 2" })).toEqual([]); + expect(getQuerySummaries([first], { traceId: "trace-b" })).toEqual([]); + }); + + it("reports resource size in bytes and retains member spans once", () => { + const first = span({ op: "resource.script", data: { "http.response_content_length": 400 } }); + const second = span({ op: "resource.script", timestamp: 1030 }); + const third = span({ op: "resource.script", data: { "http.response_content_length": "unknown" } }); + expect(getResourceSummaries([first, second, third, span()])).toEqual([ + { + description: "SELECT 1", + count: 3, + totalDurationMs: 50, + averageDurationMs: 50 / 3, + averageEncodedSizeBytes: 400 / 3, + spans: [first, second, third], + }, + ]); + }); + + it("aggregates inclusive profile frame intervals across traces and excludes the unbounded final sample", () => { + const profiles = new Map([ + ["trace-a", profile()], + ["trace-b", profile()], + ]); + expect(getProfileFunctionSummaries(profiles)).toEqual([ + { name: "app@main", totalDurationMs: 80, sampleCount: 4, traceIds: new Set(["trace-a", "trace-b"]) }, + { name: "app@query", totalDurationMs: 20, sampleCount: 2, traceIds: new Set(["trace-a", "trace-b"]) }, + ]); + expect(getProfileFunctionSummaries(profiles, { traceId: "trace-a" })).toEqual([ + { name: "app@main", totalDurationMs: 40, sampleCount: 2, traceIds: new Set(["trace-a"]) }, + { name: "app@query", totalDurationMs: 10, sampleCount: 1, traceIds: new Set(["trace-a"]) }, + ]); + expect(getProfileFunctionSummaries(profiles, { traceId: "missing" })).toEqual([]); + expect(getProfileFunctionSummaries(new Map([["empty", { ...profile(), samples: [] }]]))).toEqual([]); + expect(profiles.get("trace-a")?.samples).toHaveLength(3); + }); +}); diff --git a/packages/spotlight/src/telemetry/selectors/spans.ts b/packages/spotlight/src/telemetry/selectors/spans.ts new file mode 100644 index 000000000..a1ddb321c --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/spans.ts @@ -0,0 +1,39 @@ +import type { Span, Trace } from "../types.ts"; + +export type SpanFilter = { traceId?: string }; + +export type SpanSummary = { + description: string; + count: number; + totalDurationMs: number; + averageDurationMs: number; + spans: Span[]; +}; + +export function getSpans(traces: Iterable, filter: SpanFilter = {}): Span[] { + return Array.from(traces) + .filter(trace => filter.traceId === undefined || trace.trace_id === filter.traceId) + .flatMap(trace => Array.from(trace.spans.values())); +} + +/** Groups normalized spans by description, preserving their input order. Durations are milliseconds. */ +export function summarizeSpans(spans: Iterable): SpanSummary[] { + const groups = new Map(); + for (const span of spans) { + const description = span.description?.trim(); + if (!description || !Number.isFinite(span.timestamp) || !Number.isFinite(span.start_timestamp)) continue; + const group = groups.get(description) ?? { + description, + count: 0, + totalDurationMs: 0, + averageDurationMs: 0, + spans: [], + }; + group.count += 1; + group.totalDurationMs += span.timestamp - span.start_timestamp; + group.averageDurationMs = group.totalDurationMs / group.count; + group.spans.push(span); + groups.set(description, group); + } + return Array.from(groups.values()); +} diff --git a/packages/spotlight/src/telemetry/selectors/webVitals.ts b/packages/spotlight/src/telemetry/selectors/webVitals.ts new file mode 100644 index 000000000..9e5120e7c --- /dev/null +++ b/packages/spotlight/src/telemetry/selectors/webVitals.ts @@ -0,0 +1,14 @@ +import { PERFORMANCE_SCORE_PROFILES } from "../constants/performance.ts"; +import type { SentryEvent, SentryEventWithPerformanceData } from "../types.ts"; +import { normalizePerformanceScore } from "../utils/webVitals.ts"; + +export function getWebVitals(events: Iterable): SentryEventWithPerformanceData[] { + return Array.from(events) + .filter(event => event.measurements && event.contexts?.trace?.op === "pageload") + .map(event => { + const copy = structuredClone(event); + normalizePerformanceScore(copy, PERFORMANCE_SCORE_PROFILES); + return copy as unknown as SentryEventWithPerformanceData; + }) + .filter(event => event.measurements["score.total"] != null); +} diff --git a/packages/spotlight/src/telemetry/store/helpers.ts b/packages/spotlight/src/telemetry/store/helpers.ts new file mode 100644 index 000000000..91882215b --- /dev/null +++ b/packages/spotlight/src/telemetry/store/helpers.ts @@ -0,0 +1,6 @@ +import type { Span } from "../types.ts"; + +export function getAllSpansInTree(root: Span): Span[] { + const spans = [root]; + return root.children ? spans.concat(root.children.flatMap(getAllSpansInTree)) : spans; +} diff --git a/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts b/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts new file mode 100644 index 000000000..9bb57bc22 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/slices/envelopesSlice.ts @@ -0,0 +1,164 @@ +import type { Envelope } from "@sentry/core"; +import type { StateCreator } from "zustand/vanilla"; +import { ATTACHMENT_EVENT_TYPES, RAW_TYPES, SUPPORTED_EVENT_TYPES } from "../../constants/sentry.ts"; +import type { ProcessingRuntime } from "../../internal/types.ts"; +import type { EventAttachment, Sdk, SentryEvent } from "../../types.ts"; +import { sdkToPlatform } from "../../utils/sdkToPlatform.ts"; +import { generateUuidv4 } from "../../uuid.ts"; +import type { EnvelopesSliceActions, EnvelopesSliceState, SentryStore } from "../types.ts"; +import { normalizeSpanItem } from "../utils/spanProcessor.ts"; + +export const createEnvelopesSlice = + (runtime: ProcessingRuntime): StateCreator => + (set, get) => { + const processing = new Map[]>(); + const generatedIds = new WeakMap(); + function process(envelope: Envelope) { + const generation = runtime.generation; + const pending: Promise[] = []; + const envelopeId = String(envelope[0].__spotlight_envelope_id ?? generatedIds.get(envelope) ?? generateUuidv4()); + generatedIds.set(envelope, envelopeId); + if (get().envelopes.has(envelopeId)) + return { count: get().envelopes.size, pending: processing.get(envelopeId) ?? [] }; + processing.set(envelopeId, pending); + function finish(count: number) { + void Promise.allSettled(pending).then(() => { + if (processing.get(envelopeId) === pending) processing.delete(envelopeId); + }); + return { count, pending }; + } + const [header, items] = envelope; + const sentAt = new Date(header.sent_at as string).getTime(); + const lastSeen = Number.isFinite(sentAt) ? sentAt : runtime.receivedAtMs; + // Read the inferred source type from the envelope header (set by server) + const sourceType = (header as { __spotlight_inferred_source?: "browser" | "server" | "mobile" }) + .__spotlight_inferred_source; + let sdk: Sdk; + + if (header.sdk?.name && header.sdk.version) { + sdk = { + name: header.sdk.name, + version: header.sdk.version, + lastSeen: lastSeen, + }; + } else if (items.length > 0) { + sdk = get().inferSdkFromEvent(items[0][1] as SentryEvent); + sdk.lastSeen = lastSeen; + } else { + sdk = { + name: "unknown", + version: "0.0.0", + lastSeen, + }; + } + + get().storeSdkRecord(sdk); + if (runtime.generation !== generation) return finish(get().envelopes.size); + + const traceContext = header.trace; + + const attachmentsByEventId = new Map(); + const envelopeScopedAttachments: EventAttachment[] = []; + let eventCount = 0; + + for (const [itemHeader, itemData] of items) { + if (SUPPORTED_EVENT_TYPES.has(itemHeader.type)) { + eventCount += 1; + } + + if (!ATTACHMENT_EVENT_TYPES.has(itemHeader.type)) { + continue; + } + + const rawData = (itemData as { data?: unknown })?.data; + if (typeof rawData !== "string") { + continue; + } + + const attachment: EventAttachment = { + header: itemHeader, + data: rawData, + }; + + const eventId = "event_id" in itemHeader ? (itemHeader.event_id as string | undefined) : undefined; + + if (eventId) { + const existing = attachmentsByEventId.get(eventId) ?? []; + existing.push(attachment); + attachmentsByEventId.set(eventId, existing); + } else { + envelopeScopedAttachments.push(attachment); + } + } + + for (const [itemIndex, [itemHeader, itemData]] of items.entries()) { + if (runtime.generation !== generation) break; + if (itemHeader.type === "span") { + for (const event of normalizeSpanItem(itemData, sdk)) { + if (runtime.generation !== generation) break; + event.platform = sdkToPlatform(sdk.name); + if (sourceType) event.__sourceType = sourceType; + pending.push(get().pushEvent(event)); + } + continue; + } + if (SUPPORTED_EVENT_TYPES.has(itemHeader.type)) { + const item = structuredClone(itemData) as SentryEvent; + item.type ??= itemHeader.type as SentryEvent["type"]; + if (!RAW_TYPES.has(itemHeader.type)) { + item.platform = sdkToPlatform(sdk.name); + } + if (traceContext) { + if (!item.contexts) { + item.contexts = {}; + } + item.contexts.trace ??= traceContext; + } + // Attach the inferred source type to the event for UI display + if (sourceType) { + (item as { __sourceType?: typeof sourceType }).__sourceType = sourceType; + } + const eventId = + item.event_id ?? ("event_id" in itemHeader ? (itemHeader.event_id as string | undefined) : undefined); + item.event_id = eventId ?? `${envelopeId}:${itemIndex}`; + let attachmentsForEvent = eventId ? attachmentsByEventId.get(eventId) : undefined; + + if (!attachmentsForEvent?.length && eventCount === 1 && envelopeScopedAttachments.length > 0) { + attachmentsForEvent = envelopeScopedAttachments; + } + + if (attachmentsForEvent?.length) { + item.attachments = attachmentsForEvent; + } + pending.push(get().pushEvent(item)); + } + } + + const { envelopes } = get(); + if (runtime.generation !== generation) return finish(envelopes.size); + const newEnvelopes = new Map(envelopes); + newEnvelopes.set(envelopeId, envelope); + set({ envelopes: newEnvelopes }); + return finish(newEnvelopes.size); + } + return { + envelopes: new Map(), + // Compatibility path used by the UI: synchronous processing except optional enrichment. + pushEnvelope: envelope => { + const { count, pending } = process(envelope); + for (const task of pending) void task.catch(() => {}); + return count; + }, + ingestEnvelope: async envelope => { + const { count, pending } = process(envelope); + const results = await Promise.allSettled(pending); + const failure = results.find(result => result.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; + return count; + }, + getEnvelopeById: (id: string) => { + return get().envelopes.get(id); + }, + getEnvelopes: () => Array.from(get().envelopes.values()), + }; + }; diff --git a/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts new file mode 100644 index 000000000..5ef33c827 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts @@ -0,0 +1,241 @@ +import type { StateCreator } from "zustand/vanilla"; +import { graftProfileSpans } from "../../data/profiles.ts"; +import type { EngineOptions, ProcessingRuntime } from "../../internal/types.ts"; +import type { SentryEvent, SentryLogEventItem } from "../../types.ts"; +import type { SentryTransactionEvent } from "../../types.ts"; +import { isErrorEvent, isLogEvent, isProfileChunkEvent, isProfileEvent, isTraceEvent } from "../../utils/sentry.ts"; +import { generateUuidv4 } from "../../uuid.ts"; +import type { EventsSliceActions, EventsSliceState, SentryProfileWithTraceMeta, SentryStore } from "../types.ts"; +import { toTimestamp } from "../utils.ts"; +import { processLogItems } from "../utils/logProcessor.ts"; +import { mergeChunksToProfile, processProfileChunkEvent } from "../utils/profileChunkProcessor.ts"; +import { processProfileEvent } from "../utils/profileProcessor.ts"; +import { processSpanEvent } from "../utils/spanProcessor.ts"; +import { initializeTrace } from "../utils/traceInitializer.ts"; +import { processTransactionEvent, updateTraceMetadata } from "../utils/traceProcessor.ts"; + +export const createEventsSlice = + ( + options: EngineOptions, + runtime: ProcessingRuntime, + ): StateCreator => + (set, get) => { + const pendingIds = new Map(); + return { + eventsById: new Map(), + + pushEvent: async (input: SentryEvent & { event_id?: string }) => { + const event = structuredClone(input); + const generation = runtime.generation; + if (!event.event_id) { + event.event_id = generateUuidv4(); + } + + const { eventsById: _eventsById } = get(); + if (_eventsById.has(event.event_id) || pendingIds.get(event.event_id) === generation) return; + pendingIds.set(event.event_id, generation); + try { + if (isErrorEvent(event)) { + for (const exception of event.exception?.values ?? []) exception.stacktrace?.frames?.reverse(); + try { + if (options.enrichStacktrace) await options.enrichStacktrace(event); + } catch { + // Source context is optional; a failed adapter must not drop captured telemetry. + } + if (runtime.generation !== generation || get().eventsById.has(event.event_id)) return; + } + + event.timestamp = toTimestamp(event.timestamp ?? runtime.receivedAtMs / 1000); + if (event.start_timestamp) { + event.start_timestamp = toTimestamp(event.start_timestamp); + } + + if (event.type === "span") { + event.span.start_timestamp = toTimestamp(event.span.start_timestamp); + event.span.timestamp = toTimestamp(event.span.timestamp); + } else if (isTraceEvent(event)) { + event.spans = event.spans?.map(span => ({ + ...span, + start_timestamp: toTimestamp(span.start_timestamp), + timestamp: toTimestamp(span.timestamp), + })); + } + + if (isLogEvent(event) && event.items?.length) { + const { logsById, logsByTraceId } = get(); + const existingLogIds = new Set(logsById.keys()); + const { processedLogs } = processLogItems(event, existingLogIds); + + // Update store with processed logs + // Build up the Maps incrementally to avoid stale state capture + const newLogsById = new Map(logsById); + const newLogsByTraceId = new Map(logsByTraceId); + + for (const logItem of processedLogs) { + newLogsById.set(logItem.id, logItem); + + if (logItem.trace_id) { + const logSet = newLogsByTraceId.get(logItem.trace_id) || new Set(); + logSet.add(logItem); + newLogsByTraceId.set(logItem.trace_id, logSet); + } + } + + // Set state once with all accumulated changes + set({ logsById: newLogsById, logsByTraceId: newLogsByTraceId }); + if (runtime.generation !== generation) return; + } + + const { eventsById } = get(); + const newEventIds = new Map(eventsById); + newEventIds.set(event.event_id, event); + set({ eventsById: newEventIds }); + if (runtime.generation !== generation) return; + + // Notify event subscribers + for (const [type, callback] of get().subscribers.values()) { + if (type === "event") { + (callback as (event: SentryEvent) => void)(event); + if (runtime.generation !== generation) return; + } + } + + const traceCtx = event.contexts?.trace; + if (traceCtx?.trace_id) { + const { tracesById, profilesByTraceId, profileChunksByProfilerId } = get(); + const existingTrace = tracesById.get(traceCtx.trace_id); + + // Initialize or get existing trace + let trace = existingTrace ?? initializeTrace(event); + + // Update timestamps for all events + trace.start_timestamp = Math.min(event.start_timestamp ?? event.timestamp, trace.start_timestamp); + trace.timestamp = Math.max(event.timestamp, trace.timestamp); + + let mergedV2Profile: SentryProfileWithTraceMeta | undefined; + + if (isTraceEvent(event)) { + // Process transaction event and update trace + const result = processTransactionEvent(event, { + existingTrace: trace, + profilesByTraceId, + profileChunksByProfilerId, + }); + trace = result.trace; + mergedV2Profile = result.mergedProfile; + } else if (event.type === "span") { + const result = processSpanEvent(event, { + existingTrace: trace, + profilesByTraceId, + profileChunksByProfilerId, + }); + trace = result.trace; + mergedV2Profile = result.mergedProfile; + } else if (isErrorEvent(event)) { + // For error events, increment error count + trace.errors += 1; + } + + // Update trace metadata and status for all events with trace context + updateTraceMetadata(trace); + + if (traceCtx.status !== "ok") { + trace.status = traceCtx.status; + } + + // Always save trace to store (whether new or updated) + const newTracesById = new Map(tracesById); + newTracesById.set(trace.trace_id, trace); + + // If we merged a V2 profile, store it by trace_id so the UI can find it + if (mergedV2Profile) { + const newProfilesByTraceId = new Map(profilesByTraceId); + newProfilesByTraceId.set(trace.trace_id, mergedV2Profile); + set({ tracesById: newTracesById, profilesByTraceId: newProfilesByTraceId }); + } else { + set({ tracesById: newTracesById }); + } + } + + if (runtime.generation !== generation) return; + if (isProfileEvent(event)) { + const { profilesByTraceId, tracesById } = get(); + const { profiles } = processProfileEvent(event, { tracesById }); + + // Update store with processed profiles + const newProfilesByTraceId = new Map(profilesByTraceId); + for (const { traceId, profile } of profiles) { + newProfilesByTraceId.set(traceId, profile); + } + set({ profilesByTraceId: newProfilesByTraceId }); + } + + // Handle V2 profile chunks (continuous profiling) + if (isProfileChunkEvent(event)) { + const { profileChunksByProfilerId, tracesById, eventsById, profilesByTraceId } = get(); + const { chunk } = processProfileChunkEvent(event); + + // Update store with processed profile chunk + const newProfileChunksByProfilerId = new Map(profileChunksByProfilerId); + const existingChunks = newProfileChunksByProfilerId.get(chunk.profiler_id) || []; + + // Check if this chunk already exists (by chunk_id) + if (!existingChunks.some(c => c.chunk_id === chunk.chunk_id)) { + const updatedChunks = [...existingChunks, chunk]; + newProfileChunksByProfilerId.set(chunk.profiler_id, updatedChunks); + + // Check if there's an existing transaction with this profiler_id that needs re-grafting + // This handles the case where profile chunks arrive AFTER the transaction + let traceToUpdate: string | undefined; + let activeThreadId: string | undefined; + for (const evt of eventsById.values()) { + if (isTraceEvent(evt)) { + const txn = evt as SentryTransactionEvent; + const profileCtx = txn.contexts?.profile as { profiler_id?: string } | undefined; + if (profileCtx?.profiler_id === chunk.profiler_id) { + traceToUpdate = txn.contexts.trace.trace_id; + activeThreadId = txn.contexts.trace.data?.["thread.id"] as string | undefined; + break; + } + } + } + + if (traceToUpdate) { + // Re-graft with updated chunks + const existingTrace = tracesById.get(traceToUpdate); + if (existingTrace && !existingTrace.profileGrafted) { + const mergedProfile = mergeChunksToProfile(updatedChunks, activeThreadId); + if (mergedProfile) { + // Create a shallow copy of the trace to trigger React re-render + const trace = { + ...existingTrace, + spans: new Map(existingTrace.spans), + spanTree: [...existingTrace.spanTree], + }; + graftProfileSpans(trace, mergedProfile); + + // Store updated trace and profile + const newTracesById = new Map(tracesById); + newTracesById.set(traceToUpdate, trace); + const newProfilesByTraceId = new Map(profilesByTraceId); + newProfilesByTraceId.set(traceToUpdate, mergedProfile); + set({ + profileChunksByProfilerId: newProfileChunksByProfilerId, + tracesById: newTracesById, + profilesByTraceId: newProfilesByTraceId, + }); + return; + } + } + } + + set({ profileChunksByProfilerId: newProfileChunksByProfilerId }); + } + } + } finally { + if (pendingIds.get(event.event_id) === generation) pendingIds.delete(event.event_id); + } + }, + getEvents: () => Array.from(get().eventsById.values()), + }; + }; diff --git a/packages/spotlight/src/telemetry/store/slices/logsSlice.ts b/packages/spotlight/src/telemetry/store/slices/logsSlice.ts new file mode 100644 index 000000000..2898acef8 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/slices/logsSlice.ts @@ -0,0 +1,14 @@ +import type { StateCreator } from "zustand/vanilla"; +import type { LogsSliceActions, LogsSliceState, SentryStore } from "../types.ts"; + +export const createLogsSlice: StateCreator = (_set, get) => ({ + logsById: new Map(), + logsByTraceId: new Map(), + + getLogById: (id: string) => get().logsById.get(id), + getLogs: () => Array.from(get().logsById.values()), + getLogsByTraceId: (traceId: string) => { + const logsByTraceId = get().logsByTraceId.get(traceId); + return logsByTraceId ? Array.from(logsByTraceId) : []; + }, +}); diff --git a/packages/spotlight/src/telemetry/store/slices/profilesSlice.ts b/packages/spotlight/src/telemetry/store/slices/profilesSlice.ts new file mode 100644 index 000000000..b479bdeaf --- /dev/null +++ b/packages/spotlight/src/telemetry/store/slices/profilesSlice.ts @@ -0,0 +1,23 @@ +import type { StateCreator } from "zustand/vanilla"; +import { getProfileFunctionSummaries } from "../../selectors/profiles.ts"; +import type { ProfilesSliceActions, ProfilesSliceState, SentryStore } from "../types.ts"; + +export const createProfilesSlice: StateCreator = ( + _set, + get, +) => ({ + profilesByTraceId: new Map(), + profileChunksByProfilerId: new Map(), + + // Unified profile access - works for both V1 and merged V2 profiles + getProfileByTraceId: (id: string) => get().profilesByTraceId.get(id), + + // Keep legacy field names private to the store adapter. + getAggregateCallData: () => + getProfileFunctionSummaries(get().profilesByTraceId).map(summary => ({ + name: summary.name, + totalTime: summary.totalDurationMs, + samples: summary.sampleCount, + traceIds: summary.traceIds, + })), +}); diff --git a/packages/spotlight/src/telemetry/store/slices/sdksSlice.ts b/packages/spotlight/src/telemetry/store/slices/sdksSlice.ts new file mode 100644 index 000000000..6765ef00e --- /dev/null +++ b/packages/spotlight/src/telemetry/store/slices/sdksSlice.ts @@ -0,0 +1,39 @@ +import type { StateCreator } from "zustand/vanilla"; +import type { Sdk, SentryEvent } from "../../types.ts"; +import type { SDKsSliceActions, SDKsSliceState, SentryStore } from "../types.ts"; + +const getSDKKey = (sdk: Sdk): string => `${sdk.name}@${sdk.version}`; + +export const createSDKsSlice: StateCreator = (set, get) => ({ + sdks: new Map(), + + inferSdkFromEvent: (event: SentryEvent) => { + const sdk: Sdk = { + name: "unknown", + version: "unknown", + lastSeen: new Date().getTime(), + }; + + if (event.sdk) { + sdk.name = event.sdk.name || sdk.name; + sdk.version = event.sdk.version || sdk.version; + } else if (event.platform) { + sdk.name = event.platform; + } + + return sdk; + }, + storeSdkRecord: (sdk: Sdk) => { + const sdks = get().sdks; + const existingSdk = sdks.get(getSDKKey(sdk)); + if (existingSdk) { + existingSdk.lastSeen = sdk.lastSeen; + } else { + const newSdks = new Map(sdks); + newSdks.set(getSDKKey(sdk), sdk); + set({ sdks: newSdks }); + } + return existingSdk || sdk; + }, + getSdks: () => Array.from(get().sdks.values()), +}); diff --git a/packages/spotlight/src/telemetry/store/slices/sharedSlice.ts b/packages/spotlight/src/telemetry/store/slices/sharedSlice.ts new file mode 100644 index 000000000..60602c299 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/slices/sharedSlice.ts @@ -0,0 +1,30 @@ +import type { StateCreator } from "zustand/vanilla"; +import type { ProcessingRuntime } from "../../internal/types.ts"; +import type { SentryStore, SharedSliceActions } from "../types.ts"; + +export const createSharedSlice = + (runtime: ProcessingRuntime): StateCreator => + (set, get) => ({ + getEventById: (id: string) => get().eventsById.get(id), + getTraceById: (id: string) => get().tracesById.get(id), + getEventsByTrace: (traceId: string, spanId?: string | null) => + get() + .getEvents() + .filter(event => { + const trace = event.contexts?.trace; + return trace?.trace_id === traceId && (spanId === undefined || trace.span_id === spanId); + }), + resetData: () => { + runtime.generation += 1; + set({ + envelopes: new Map(), + eventsById: new Map(), + tracesById: new Map(), + sdks: new Map(), + profilesByTraceId: new Map(), + profileChunksByProfilerId: new Map(), + logsById: new Map(), + logsByTraceId: new Map(), + }); + }, + }); diff --git a/packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts b/packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts new file mode 100644 index 000000000..3a245e833 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/slices/subscriptionsSlice.ts @@ -0,0 +1,27 @@ +import type { StateCreator } from "zustand/vanilla"; +import { generateUuidv4 } from "../../uuid.ts"; +import type { SentryStore, Subscription, SubscriptionsSliceActions, SubscriptionsSliceState } from "../types.ts"; + +export const createSubscriptionsSlice: StateCreator< + SentryStore, + [], + [], + SubscriptionsSliceState & SubscriptionsSliceActions +> = (set, get) => ({ + subscribers: new Map(), + + subscribe: (...args: Subscription) => { + const id = generateUuidv4(); + const { subscribers } = get(); + const newSubscribers = new Map(subscribers); + newSubscribers.set(id, args); + set({ subscribers: newSubscribers }); + + return () => { + const { subscribers: currentSubscribers } = get(); + const updatedSubscribers = new Map(currentSubscribers); + updatedSubscribers.delete(id); + set({ subscribers: updatedSubscribers }); + }; + }, +}); diff --git a/packages/spotlight/src/telemetry/store/slices/tracesSlice.ts b/packages/spotlight/src/telemetry/store/slices/tracesSlice.ts new file mode 100644 index 000000000..3e44b5795 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/slices/tracesSlice.ts @@ -0,0 +1,11 @@ +import type { StateCreator } from "zustand/vanilla"; +import type { SentryStore, TracesSliceActions, TracesSliceState } from "../types.ts"; + +export const createTracesSlice: StateCreator = ( + _, + get, +) => ({ + tracesById: new Map(), + + getTraces: () => Array.from(get().tracesById.values()), +}); diff --git a/packages/spotlight/src/telemetry/store/types.ts b/packages/spotlight/src/telemetry/store/types.ts new file mode 100644 index 000000000..cb761069a --- /dev/null +++ b/packages/spotlight/src/telemetry/store/types.ts @@ -0,0 +1,107 @@ +import type { Envelope } from "@sentry/core"; +import type { AggregateCallData, Sdk, SentryEvent, SentryLogEventItem, Trace } from "../types.ts"; +import type { ProcessedProfileChunk } from "./utils/profileChunkProcessor.ts"; + +import type { SentryProfileWithTraceMeta } from "../types.ts"; +export type { SentryProfileWithTraceMeta } from "../types.ts"; + +export type OnlineSubscription = ["online", (status: boolean) => void]; +export type EventSubscription = ["event", (event: SentryEvent) => void]; +export type TraceSubscription = ["trace", (trace: Trace) => void]; +export type Subscription = OnlineSubscription | EventSubscription | TraceSubscription; + +export interface EventsSliceState { + eventsById: Map; +} + +export interface EventsSliceActions { + pushEvent: (event: SentryEvent & { event_id?: string }) => Promise; + getEvents: () => SentryEvent[]; +} + +export interface TracesSliceState { + tracesById: Map; +} + +export interface TracesSliceActions { + getTraces: () => Trace[]; +} + +export interface ProfilesSliceState { + /** Final merged profiles keyed by trace_id - works for both V1 and V2 */ + profilesByTraceId: Map; + /** Internal: V2 chunks being accumulated, keyed by profiler_id */ + profileChunksByProfilerId: Map; +} + +export interface ProfilesSliceActions { + /** Get profile for a trace - unified access for both V1 and V2 profiles */ + getProfileByTraceId: (id: string) => SentryProfileWithTraceMeta | undefined; + getAggregateCallData: () => AggregateCallData[]; +} + +export interface SubscriptionsSliceState { + subscribers: Map; +} + +export interface SubscriptionsSliceActions { + subscribe: (...args: Subscription) => () => void; +} + +export interface EnvelopesSliceState { + envelopes: Map; +} + +export interface EnvelopesSliceActions { + pushEnvelope: (params: Envelope) => number; + ingestEnvelope: (params: Envelope) => Promise; + getEnvelopeById: (id: string) => Envelope | undefined; + getEnvelopes: () => Array; +} + +export interface SDKsSliceState { + sdks: Map; +} + +export interface LogsSliceState { + logsById: Map; + logsByTraceId: Map>; +} + +export interface LogsSliceActions { + getLogById: (id: string) => SentryLogEventItem | undefined; + getLogs: () => SentryLogEventItem[]; + getLogsByTraceId: (traceId: string) => SentryLogEventItem[]; +} + +export interface SDKsSliceActions { + inferSdkFromEvent: (event: SentryEvent) => Sdk; + storeSdkRecord: (sdk: Sdk) => Sdk; + getSdks: () => Sdk[]; +} + +export interface SharedSliceActions { + getEventById: (id: string) => SentryEvent | undefined; + getTraceById: (id: string) => Trace | undefined; + getEventsByTrace: (traceId: string, spanId?: string | null) => SentryEvent[]; + resetData: () => void; +} + +export type SentryStoreState = EventsSliceState & + TracesSliceState & + ProfilesSliceState & + SubscriptionsSliceState & + EnvelopesSliceState & + LogsSliceState & + SDKsSliceState; + +export type SentryStoreActions = EventsSliceActions & + TracesSliceActions & + ProfilesSliceActions & + SubscriptionsSliceActions & + EnvelopesSliceActions & + LogsSliceActions & + SDKsSliceActions & + SharedSliceActions; + +export type SentryStore = SentryStoreState & SentryStoreActions; diff --git a/packages/spotlight/src/telemetry/store/utils.ts b/packages/spotlight/src/telemetry/store/utils.ts new file mode 100644 index 000000000..926c9c536 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/utils.ts @@ -0,0 +1,9 @@ +export function toTimestamp(date: string | number) { + if (typeof date === "string") return new Date(date).getTime(); + return date * 1000; +} + +export function relativeNsToTimestamp(startTs: number, ns: number | string) { + const nsStr = ns.toString(); + return nsStr.length > 3 ? startTs + Number.parseInt(nsStr.slice(0, -3), 10) / 1000 : startTs; +} diff --git a/packages/spotlight/src/telemetry/store/utils/logProcessor.ts b/packages/spotlight/src/telemetry/store/utils/logProcessor.ts new file mode 100644 index 000000000..6a68812b7 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/utils/logProcessor.ts @@ -0,0 +1,45 @@ +import type { SentryLogEvent, SentryLogEventItem } from "../../types.ts"; +import { toTimestamp } from "../utils.ts"; + +export interface LogProcessingResult { + processedLogs: SentryLogEventItem[]; +} + +/** + * Processes log items from a log event, normalizing and enriching them with metadata. + * @param event The log event to process + * @param existingLogIds Set of already processed log IDs to avoid duplicates + * @returns Processed log items ready to be stored + */ +export function processLogItems(event: SentryLogEvent, existingLogIds: Set): LogProcessingResult { + const processedLogs: SentryLogEventItem[] = []; + + if (!event.items?.length) { + return { processedLogs }; + } + + for (const [index, logItem] of event.items.entries()) { + const logId = logItem.id || `${event.event_id}:log:${index}`; + + // Skip if already processed + if (existingLogIds.has(logId)) { + continue; + } + + // Normalize severity number + if (logItem.severity_number == null) { + logItem.severity_number = 0; + } + + // Extract SDK name from attributes + logItem.sdk = logItem.attributes?.["sentry.sdk.name"]?.value as string; + + // Normalize timestamp + logItem.timestamp = toTimestamp(logItem.timestamp); + logItem.id = logId; + + processedLogs.push(logItem); + } + + return { processedLogs }; +} diff --git a/packages/spotlight/src/telemetry/store/utils/profileChunkProcessor.ts b/packages/spotlight/src/telemetry/store/utils/profileChunkProcessor.ts new file mode 100644 index 000000000..6aa72a485 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/utils/profileChunkProcessor.ts @@ -0,0 +1,157 @@ +import type { ProcessedProfileSample, SentryProfileV2ChunkEvent } from "../../types.ts"; +import type { SentryProfileWithTraceMeta } from "../types.ts"; + +/** + * Represents a processed V2 profile chunk ready for storage and visualization. + * V2 chunks use absolute timestamps and are keyed by profiler_id. + */ +export type ProcessedProfileChunk = { + profiler_id: string; + chunk_id: string; + platform: string; + release?: string; + environment?: string; + samples: ProcessedProfileSample[]; + stacks: number[][]; + frames: SentryProfileV2ChunkEvent["profile"]["frames"]; + thread_metadata: SentryProfileV2ChunkEvent["profile"]["thread_metadata"]; + // Timestamps derived from samples + start_timestamp: number; + end_timestamp: number; +}; + +export interface ProfileChunkProcessingResult { + chunk: ProcessedProfileChunk; +} + +/** + * Processes a V2 profile chunk event (continuous profiling). + * V2 samples use absolute Unix timestamps, so no relative-to-start conversion needed. + */ +export function processProfileChunkEvent(event: SentryProfileV2ChunkEvent): ProfileChunkProcessingResult { + const { profile, profiler_id, chunk_id, platform, release, environment } = event; + + // Convert V2 samples to processed format + // V2 uses absolute Unix timestamps in seconds with microsecond precision + const processedSamples: ProcessedProfileSample[] = profile.samples.map(sample => ({ + stack_id: sample.stack_id, + thread_id: sample.thread_id, + start_timestamp: sample.timestamp * 1000, // Convert to milliseconds for internal use + })); + + // Sort samples by timestamp + processedSamples.sort((a, b) => a.start_timestamp - b.start_timestamp); + + // Calculate chunk time boundaries from samples + const start_timestamp = processedSamples.length > 0 ? processedSamples[0].start_timestamp : Date.now(); + const end_timestamp = + processedSamples.length > 0 ? processedSamples[processedSamples.length - 1].start_timestamp : start_timestamp; + + // Reverse stacks to match the order expected by visualization (leaf first) + const reversedStacks = profile.stacks.map((s: number[]) => Array.from(s).reverse()); + + const chunk: ProcessedProfileChunk = { + profiler_id, + chunk_id, + platform, + release, + environment, + samples: processedSamples, + stacks: reversedStacks, + frames: profile.frames, + thread_metadata: profile.thread_metadata, + start_timestamp, + end_timestamp, + }; + + return { chunk }; +} + +/** + * Merges multiple profile chunks from the same profiler session into a single profile + * that can be used with existing profile visualization components. + */ +export function mergeChunksToProfile( + chunks: ProcessedProfileChunk[], + activeThreadId?: string, +): SentryProfileWithTraceMeta | null { + if (chunks.length === 0) return null; + + // Sort chunks by start timestamp + const sortedChunks = [...chunks].sort((a, b) => a.start_timestamp - b.start_timestamp); + + // Merge frames and build frame index mapping + const mergedFrames: ProcessedProfileChunk["frames"] = []; + const frameIndexMaps: Map[] = []; + + for (const chunk of sortedChunks) { + const frameMap = new Map(); + for (let i = 0; i < chunk.frames.length; i++) { + // For simplicity, we add all frames and dedupe by reference + // A more sophisticated approach would dedupe by frame content + const newIndex = mergedFrames.length; + mergedFrames.push(chunk.frames[i]); + frameMap.set(i, newIndex); + } + frameIndexMaps.push(frameMap); + } + + // Merge stacks with remapped frame indices + const mergedStacks: number[][] = []; + const stackIndexMaps: Map[] = []; + + for (let chunkIdx = 0; chunkIdx < sortedChunks.length; chunkIdx++) { + const chunk = sortedChunks[chunkIdx]; + const frameMap = frameIndexMaps[chunkIdx]; + const stackMap = new Map(); + + for (let i = 0; i < chunk.stacks.length; i++) { + const remappedStack = chunk.stacks[i].map(frameIdx => frameMap.get(frameIdx) ?? frameIdx); + const newIndex = mergedStacks.length; + mergedStacks.push(remappedStack); + stackMap.set(i, newIndex); + } + stackIndexMaps.push(stackMap); + } + + // Merge samples with remapped stack indices + const mergedSamples: ProcessedProfileSample[] = []; + for (let chunkIdx = 0; chunkIdx < sortedChunks.length; chunkIdx++) { + const chunk = sortedChunks[chunkIdx]; + const stackMap = stackIndexMaps[chunkIdx]; + + for (const sample of chunk.samples) { + mergedSamples.push({ + ...sample, + stack_id: stackMap.get(sample.stack_id) ?? sample.stack_id, + }); + } + } + + // Sort all samples by timestamp + mergedSamples.sort((a, b) => a.start_timestamp - b.start_timestamp); + + // Merge thread metadata + const mergedThreadMetadata: ProcessedProfileChunk["thread_metadata"] = {}; + for (const chunk of sortedChunks) { + Object.assign(mergedThreadMetadata, chunk.thread_metadata); + } + + // Determine active thread ID - use provided or first thread with samples + const resolvedActiveThreadId = + activeThreadId || + (mergedSamples.length > 0 ? mergedSamples[0].thread_id : Object.keys(mergedThreadMetadata)[0] || "0"); + + // Calculate overall timestamp from first chunk + const timestamp = sortedChunks[0].start_timestamp; + + return { + platform: sortedChunks[0].platform, + thread_metadata: mergedThreadMetadata, + samples: mergedSamples, + frames: mergedFrames, + stacks: mergedStacks, + timestamp, + active_thread_id: resolvedActiveThreadId, + }; +} diff --git a/packages/spotlight/src/telemetry/store/utils/profileProcessor.ts b/packages/spotlight/src/telemetry/store/utils/profileProcessor.ts new file mode 100644 index 000000000..f5825f5c2 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/utils/profileProcessor.ts @@ -0,0 +1,75 @@ +import { graftProfileSpans } from "../../data/profiles.ts"; +import type { ProfileSample, SentryProfileTransactionInfo, SentryProfileV1Event, Trace } from "../../types.ts"; +import type { SentryProfileWithTraceMeta } from "../types.ts"; +import { relativeNsToTimestamp } from "../utils.ts"; + +export interface ProfileProcessingContext { + tracesById: Map; +} + +export interface ProfileProcessingResult { + profiles: Array<{ + traceId: string; + profile: SentryProfileWithTraceMeta; + }>; +} + +/** + * Processes a profile event and creates profile data for each transaction. + * Handles profile grafting onto existing traces if available. + */ +export function processProfileEvent( + event: SentryProfileV1Event, + context: ProfileProcessingContext, +): ProfileProcessingResult { + const { tracesById } = context; + const profiles: ProfileProcessingResult["profiles"] = []; + + // Ensure transactions array exists + if (!event.transactions) { + event.transactions = event.transaction ? [event.transaction] : []; + } + + for (const txn of event.transactions) { + // Skip string transaction IDs + if (typeof txn === "string") continue; + + const profileTxn = txn as SentryProfileTransactionInfo; + const trace = tracesById.get(profileTxn.trace_id); + + // Calculate timestamp + const timestamp = + trace && profileTxn.relative_start_ns != null + ? relativeNsToTimestamp(trace.start_timestamp, profileTxn.relative_start_ns) + : event.timestamp; + + // Build profile data + const profile: SentryProfileWithTraceMeta = { + platform: event.platform, + thread_metadata: event.profile.thread_metadata, + samples: event.profile.samples.map((s: ProfileSample) => ({ + stack_id: s.stack_id, + thread_id: s.thread_id, + elapsed_since_start_ns: s.elapsed_since_start_ns, + start_timestamp: relativeNsToTimestamp(timestamp, s.elapsed_since_start_ns), + })), + frames: event.profile.frames, + stacks: event.profile.stacks.map((s: number[]) => Array.from(s).reverse()), + timestamp, + active_thread_id: profileTxn.active_thread_id, + }; + + profiles.push({ + traceId: profileTxn.trace_id, + profile, + }); + + // Graft profile onto trace if it's ready + // Avoid grafting partial traces (where we mocked start_timestamp from event timestamp) + if (trace && trace.start_timestamp < trace.timestamp) { + graftProfileSpans(trace, profile); + } + } + + return { profiles }; +} diff --git a/packages/spotlight/src/telemetry/store/utils/spanProcessor.ts b/packages/spotlight/src/telemetry/store/utils/spanProcessor.ts new file mode 100644 index 000000000..5c00735ce --- /dev/null +++ b/packages/spotlight/src/telemetry/store/utils/spanProcessor.ts @@ -0,0 +1,106 @@ +import type { Sdk, SentrySpanEvent, SentryTransactionEvent, Span } from "../../types.ts"; +import { initializeTrace } from "./traceInitializer.ts"; +import { + type TraceProcessingContext, + type TraceProcessingResult, + processTransactionEvent, + rebuildSpanTree, +} from "./traceProcessor.ts"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** Decode the span v2 container, or a legacy standalone span, without changing the raw envelope. */ +export function normalizeSpanItem(payload: unknown, sdk: Sdk): SentrySpanEvent[] { + if (!isRecord(payload)) return []; + const isContainer = "items" in payload; + if (isContainer && (payload.version !== 2 || !Array.isArray(payload.items))) return []; + + const events: SentrySpanEvent[] = []; + for (const raw of isContainer ? (payload.items as unknown[]) : [payload]) { + if (!isRecord(raw)) continue; + const end = isContainer ? raw.end_timestamp : raw.timestamp; + if ( + typeof raw.trace_id !== "string" || + !raw.trace_id || + typeof raw.span_id !== "string" || + !raw.span_id || + typeof raw.start_timestamp !== "number" || + !Number.isFinite(raw.start_timestamp) || + typeof end !== "number" || + !Number.isFinite(end) + ) + continue; + + const data: Record = {}; + if (isContainer && isRecord(raw.attributes)) { + for (const [key, attribute] of Object.entries(raw.attributes)) { + if (isRecord(attribute) && "value" in attribute) data[key] = attribute.value; + } + } else if (isRecord(raw.data)) { + Object.assign(data, raw.data); + } + + const op = isContainer ? data["sentry.op"] : raw.op; + const name = isContainer ? raw.name : raw.description; + const span: Span = { + trace_id: raw.trace_id, + span_id: raw.span_id, + parent_span_id: typeof raw.parent_span_id === "string" ? raw.parent_span_id : undefined, + description: typeof name === "string" ? name : undefined, + op: typeof op === "string" ? op : undefined, + start_timestamp: raw.start_timestamp, + timestamp: end, + status: typeof raw.status === "string" ? raw.status : undefined, + data, + }; + + events.push({ + type: "span", + event_id: `${raw.trace_id}:${raw.span_id}`, + timestamp: end, + start_timestamp: raw.start_timestamp, + contexts: { + trace: { + trace_id: raw.trace_id, + span_id: raw.span_id, + parent_span_id: span.parent_span_id ?? undefined, + op: span.op ?? undefined, + status: span.status, + data, + }, + }, + sdk, + span, + is_segment: raw.is_segment === true, + }); + } + return events; +} + +export function processSpanEvent(event: SentrySpanEvent, context: TraceProcessingContext): TraceProcessingResult { + const trace = context.existingTrace ?? initializeTrace(event); + const span = { + ...event.span, + start_timestamp: event.span.start_timestamp, + timestamp: event.span.timestamp, + }; + + // A segment serves the same role as the root transaction in the existing UI. + // Child spans must not inflate the transaction count or replace its name. + if (event.is_segment) { + const transaction: SentryTransactionEvent = { + ...event, + type: "transaction", + transaction: span.description ?? "(unknown transaction)", + start_timestamp: event.start_timestamp as SentryTransactionEvent["start_timestamp"], + contexts: { ...event.contexts, trace: event.contexts!.trace! }, + }; + return processTransactionEvent(transaction, { ...context, existingTrace: trace }); + } + + trace.spans.set(span.span_id, span); + rebuildSpanTree(trace); + return { trace, shouldUpdateTrace: true }; +} diff --git a/packages/spotlight/src/telemetry/store/utils/traceInitializer.ts b/packages/spotlight/src/telemetry/store/utils/traceInitializer.ts new file mode 100644 index 000000000..c5d0771e6 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/utils/traceInitializer.ts @@ -0,0 +1,30 @@ +import type { SentryEvent, SentryTransactionEvent, Span, Trace } from "../../types.ts"; + +/** + * Creates a new trace object from an event's trace context. + * This is used when we receive an event with a trace_id but don't have a trace yet. + */ +export function initializeTrace(event: SentryEvent): Trace { + // Add guard to ensure contexts, trace context and trace_id exist + if (!event.contexts?.trace || !event.contexts.trace.trace_id) { + throw new Error("Event missing required trace context or trace_id"); + } + + const traceCtx = event.contexts.trace; + const timestamp = event.timestamp ?? 0; + + return { + ...traceCtx, + trace_id: traceCtx.trace_id!, + spans: new Map(), + spanTree: [] as Span[], + transactions: [] as SentryTransactionEvent[], + errors: 0, + start_timestamp: event.start_timestamp ?? timestamp, + timestamp, + status: traceCtx.status, + rootTransactionName: event.transaction || "(unknown transaction)", + rootTransaction: null, + profileGrafted: false, + }; +} diff --git a/packages/spotlight/src/telemetry/store/utils/traceProcessor.ts b/packages/spotlight/src/telemetry/store/utils/traceProcessor.ts new file mode 100644 index 000000000..9e7b5ce29 --- /dev/null +++ b/packages/spotlight/src/telemetry/store/utils/traceProcessor.ts @@ -0,0 +1,187 @@ +import { graftProfileSpans } from "../../data/profiles.ts"; +import { log } from "../../logger.ts"; +import type { SentryTransactionEvent, Span, Trace } from "../../types.ts"; +import { compareSpans, groupSpans } from "../../utils/traces.ts"; +import type { SentryProfileWithTraceMeta } from "../types.ts"; +import type { ProcessedProfileChunk } from "./profileChunkProcessor.ts"; +import { mergeChunksToProfile } from "./profileChunkProcessor.ts"; + +export interface TraceProcessingContext { + existingTrace?: Trace; + profilesByTraceId: Map; + profileChunksByProfilerId?: Map; +} + +export interface TraceProcessingResult { + trace: Trace; + shouldUpdateTrace: boolean; + /** V2 profile merged from chunks, to be stored in profilesByTraceId */ + mergedProfile?: SentryProfileWithTraceMeta; +} + +/** + * Processes a transaction event and updates the trace data structure. + * Handles span tree construction, profile grafting, and trace metadata. + */ +export function processTransactionEvent( + event: SentryTransactionEvent, + context: TraceProcessingContext, +): TraceProcessingResult { + const traceCtx = event.contexts.trace; + const { existingTrace, profilesByTraceId, profileChunksByProfilerId } = context; + + // Add guard to ensure trace_id exists + if (!traceCtx.trace_id) { + throw new Error("Transaction event missing required trace_id"); + } + + // Initialize or get existing trace + const trace: Trace = existingTrace ?? { + ...traceCtx, + trace_id: traceCtx.trace_id, + spans: new Map(), + spanTree: [] as Span[], + transactions: [] as SentryTransactionEvent[], + errors: 0, + start_timestamp: event.start_timestamp ?? event.timestamp, + timestamp: event.timestamp, + status: traceCtx.status, + rootTransactionName: event.transaction || "(unknown transaction)", + rootTransaction: null, + profileGrafted: false, + }; + + // Update trace timestamps + trace.start_timestamp = Math.min(event.start_timestamp ?? event.timestamp, trace.start_timestamp); + trace.timestamp = Math.max(event.timestamp, trace.timestamp); + + // Legacy transactions and streamed segments can describe the same span. + const previousIndex = trace.transactions.findIndex(txn => txn.contexts.trace.span_id === traceCtx.span_id); + if (traceCtx.span_id && previousIndex !== -1) { + const previous = trace.transactions[previousIndex]; + trace.transactions[previousIndex] = { + ...previous, + ...event, + transaction: event.transaction ?? previous.transaction, + start_timestamp: Math.min(previous.start_timestamp, event.start_timestamp), + timestamp: Math.max(previous.timestamp, event.timestamp), + contexts: { + ...previous.contexts, + ...event.contexts, + trace: { + ...previous.contexts.trace, + ...Object.fromEntries(Object.entries(event.contexts.trace).filter(([, value]) => value !== undefined)), + data: { ...previous.contexts.trace.data, ...event.contexts.trace.data }, + }, + }, + spans: Array.from( + new Map([...(previous.spans ?? []), ...(event.spans ?? [])].map(span => [span.span_id, span])).values(), + ), + }; + } else { + trace.transactions.push(event); + } + trace.transactions.sort(compareSpans); + + // Keep standalone spans received before this transaction/segment finished. + const spanMap: Map = new Map(trace.spans); + for (const txn of trace.transactions) { + const txnTrace = txn.contexts.trace; + if (!txnTrace || !txnTrace.span_id || !txnTrace.trace_id) { + continue; + } + + spanMap.set(txnTrace.span_id, { + ...txnTrace, + span_id: txnTrace.span_id, + trace_id: txnTrace.trace_id, + tags: txn?.tags, + start_timestamp: txn.start_timestamp, + timestamp: txn.timestamp, + description: traceCtx.description || txn.transaction, + transaction: txn, + }); + + // Add child spans + if (txn.spans) { + for (const span of txn.spans) { + spanMap.set(span.span_id, { + ...span, + timestamp: span.timestamp, + start_timestamp: span.start_timestamp, + }); + } + } + } + + trace.spans = spanMap; + rebuildSpanTree(trace); + + // Graft profile data if available (V1 profiles keyed by trace_id) + const profileForTrace = profilesByTraceId.get(trace.trace_id); + if (profileForTrace) { + graftProfileSpans(trace, profileForTrace); + } + + // Try V2 continuous profiling: look for profiler_id in transaction context + // V2 uses contexts.profile.profiler_id to link profiles + let mergedProfile: SentryProfileWithTraceMeta | undefined; + if (!trace.profileGrafted && profileChunksByProfilerId) { + const profileContext = event.contexts?.profile as { profiler_id?: string } | undefined; + const profilerId = profileContext?.profiler_id; + + if (profilerId) { + const chunks = profileChunksByProfilerId.get(profilerId); + if (chunks && chunks.length > 0) { + // Get the active thread ID from trace context data if available + const activeThreadId = traceCtx.data?.["thread.id"] as string | undefined; + mergedProfile = mergeChunksToProfile(chunks, activeThreadId) ?? undefined; + if (mergedProfile) { + graftProfileSpans(trace, mergedProfile); + } + } + } + } + + return { + trace, + shouldUpdateTrace: true, + mergedProfile, + }; +} + +/** Rebuild from real spans so late parents replace placeholders and children are not appended twice. */ +export function rebuildSpanTree(trace: Trace): void { + trace.spans = new Map( + Array.from(trace.spans) + .filter(([, span]) => span.op !== "orphan") + .map(([id, span]) => [id, { ...span, children: undefined }]), + ); + trace.spanTree = groupSpans(trace.spans); +} + +/** + * Updates trace metadata based on its transactions. + * Determines root transaction and handles orphan traces. + */ +export function updateTraceMetadata(trace: Trace): void { + const roots = trace.transactions.filter(e => !e.contexts.trace.parent_span_id); + + if (roots.length === 1) { + trace.rootTransaction = roots[0]; + trace.rootTransactionName = roots[0].transaction || "(unknown transaction)"; + } else if (roots.length > 1) { + trace.rootTransactionName = "(multiple root transactions)"; + } else if (trace.transactions.length > 0) { + // Orphan trace: no root transaction, but has child transactions + log( + `[Telemetry] Orphan trace detected (trace_id: ${trace.trace_id}). ` + + `Using first transaction "${trace.transactions[0].transaction}" as fallback.`, + ); + // use the first transcation for orphan traces + trace.rootTransaction = trace.transactions[0]; + trace.rootTransactionName = trace.transactions[0].transaction || "(orphan transaction)"; + } else { + trace.rootTransactionName = "(missing root transaction)"; + } +} diff --git a/packages/spotlight/src/telemetry/types.ts b/packages/spotlight/src/telemetry/types.ts new file mode 100644 index 000000000..9fffab320 --- /dev/null +++ b/packages/spotlight/src/telemetry/types.ts @@ -0,0 +1,484 @@ +import type { EnvelopeItem, EventEnvelopeHeaders, Measurements, SerializedLog } from "@sentry/core"; + +export type TraceId = string; +export type SpanId = string; + +export type EventAttachment = { + header: EnvelopeItem[0]; + data: string; +}; + +export type FrameVars = { + [key: string]: string; +}; + +export type EventFrame = { + filename?: string; + abs_path?: string; + function?: string; + module?: string; + lineno?: number; + colno?: number; + pre_context?: string[]; + post_context?: string[]; + context_line?: string; + vars?: FrameVars; + instruction_addr?: string; + in_app?: boolean; +}; + +export type EventStacktrace = { + frames: EventFrame[]; +}; + +export type EventExceptionValue = { + type: string; + value: string; + stacktrace?: EventStacktrace; +}; + +export type EventException = + | { + values: EventExceptionValue[]; + value: undefined; + } + | { + values: undefined; + value: EventExceptionValue; + }; + +export type Breadcrumb = { + data?: Record; + message?: string; + category: string; + timestamp: string; + type: string | "default"; +}; + +export type Context = Record; + +type CommonEventAttrs = { + // not always present, but we are forcing it in EventCache + event_id: string; + timestamp: number; + message?: SentryFormattedMessage; + breadcrumbs?: Breadcrumb[] | { values: Breadcrumb[] }; + transaction?: string; + environment?: string; + platform?: string; + server_name?: string; + release?: string; + start_timestamp?: number; + contexts?: Contexts; + tags?: Tags; + extra?: Context; + request?: Record | string>; + modules?: Record; + sdk?: Sdk; + measurements?: Measurements; + attachments?: EventAttachment[]; + // Inferred source type for distinguishing browser/server/mobile events + __sourceType?: "browser" | "server" | "mobile"; +}; + +// Note: For some reason the `sentry/core` module doesn't have these additional properties +// in `EventEnvelopeHeaders['trace']` but they are present in the actual events. +// Follow up? +export type TraceContext = EventEnvelopeHeaders["trace"] & { + span_id?: string; + status?: "ok" | string; + description?: string; + parent_span_id?: string; + data?: Record; + op?: string; +}; + +export type FeedbackContext = { + message: string; + name?: string; + contact_email?: string; + source?: string; + url?: string; + associated_event_id?: string; + replay_id?: string; +}; + +export type Contexts = { + runtime?: Context; + browser?: Context; + trace?: TraceContext; + feedback?: FeedbackContext; +} & { + [key: string]: Context | TraceContext | FeedbackContext | undefined; +}; + +export type Tags = { + [key: string]: string; +}; + +export type SentryFormattedMessage = + | string + | { + formatted: string; + params?: []; + }; + +export type SentryErrorEvent = CommonEventAttrs & { + type?: "error" | "event" | "message" | "default"; + level?: "fatal" | "error" | "warning" | "info" | "debug"; + exception: EventException; +}; + +export type Span = { + trace_id?: TraceId; + span_id: SpanId; + parent_span_id?: string | null; + op?: string | null; + description?: string | null; + start_timestamp: number; + tags?: Tags | null; + timestamp: number; + status?: "ok" | string; + transaction?: SentryTransactionEvent; + children?: Span[]; + data?: Record; +}; + +export type SentryTransactionEvent = CommonEventAttrs & { + type: "transaction"; + spans?: Span[]; + start_timestamp: number; + contexts: Contexts & { + trace: TraceContext; + }; +}; + +// Standalone spans are normalized at the envelope boundary. Only segment spans +// are represented as transactions; child spans remain individual spans. +export type SentrySpanEvent = CommonEventAttrs & { + type: "span"; + span: Span; + is_segment: boolean; +}; + +export type ProfileSample = { + elapsed_since_start_ns: string; + stack_id: number; + thread_id: string; +}; + +export type ProcessedProfileSample = { + start_timestamp: number; + stack_id: number; + thread_id: string; +}; + +export type SentryProfile = { + samples: ProfileSample[]; + stacks: number[][]; + frames: EventFrame[]; + platform?: string; + thread_metadata?: Record< + string, + { + name?: string; + priority?: number; + } + >; +}; + +export type SentryProcessedProfile = Omit & { + samples: ProcessedProfileSample[]; +}; + +export type SentryProfileWithTraceMeta = SentryProcessedProfile & { + timestamp: number; + active_thread_id: string; +}; + +export type AggregateCallData = { + name: string; + totalTime: number; + samples: number; + traceIds: Set; +}; + +export type SentryDeviceInfo = { + architecture: string; + is_emulator?: boolean; + locale?: string; + manufacturer?: string; + model?: string; +}; + +export type SentryOSInfo = { + name: string; + version: string; + build_number?: string; +}; + +export type SentryProfileTransactionInfo = { + name: string; + id: string; + trace_id: string; + active_thread_id: string; + relative_start_ns?: string; + relative_end_ns?: string; +}; + +export type SentryProfileV1Event = CommonEventAttrs & { + type: "profile"; + device: SentryDeviceInfo; + os: SentryOSInfo; + transactions?: Array; + transaction?: SentryProfileTransactionInfo; + version: "1"; + profile: SentryProfile; +}; + +// V2 Profile (Continuous Profiling) Types +export type ProfileV2Sample = { + timestamp: number; // Unix timestamp in seconds with microseconds precision + stack_id: number; + thread_id: string; +}; + +export type ProfileV2MeasurementValue = { + timestamp: number; + value: number; +}; + +export type ProfileV2Measurement = { + unit: string; + values: ProfileV2MeasurementValue[]; +}; + +export type SentryProfileV2 = { + samples: ProfileV2Sample[]; + stacks: number[][]; + frames: EventFrame[]; + thread_metadata: Record< + string, + { + name?: string; + priority?: number; + } + >; +}; + +export type SentryProfileV2ChunkEvent = Omit, "measurements"> & + Pick & { + type: "profile_chunk"; + version: "2"; + profiler_id: string; // Links chunks from same profiler session + chunk_id: string; // Unique ID for this chunk + platform: string; + release?: string; + environment?: string; + client_sdk?: { + name: string; + version: string; + }; + debug_meta?: { + images?: Array<{ + debug_id?: string; + image_addr?: string; + type?: string; + image_size?: number; + code_file?: string; + }>; + }; + measurements?: Record; + profile: SentryProfileV2; + }; + +export type SentryLogEventItem = SerializedLog & { + id: string; // Need to have a unique id for each log + severity_number: number; + sdk: string | undefined; +}; + +export type SentryLogEvent = CommonEventAttrs & { + type: "log"; + items: Array; +}; + +export type SentryFeedbackEvent = CommonEventAttrs & { + type: "feedback"; + contexts: Contexts & { + feedback: FeedbackContext; + }; +}; + +export type SentryEvent = + | SentryErrorEvent + | SentryTransactionEvent + | SentrySpanEvent + | SentryProfileV1Event + | SentryProfileV2ChunkEvent + | SentryLogEvent + | SentryFeedbackEvent; + +export type Trace = TraceContext & { + trace_id: string; + transactions: SentryTransactionEvent[]; + errors: number; + start_timestamp: number; + timestamp: number; + status?: "ok" | string; + rootTransaction: SentryTransactionEvent | null; + rootTransactionName: string; + spans: Map; + spanTree: Span[]; + profileGrafted: boolean; +}; + +export type Sdk = { + name: string; + version: string; + lastSeen: number; +}; + +export type SentryEventWithPerformanceData = Omit & { + measurements: Record< + string, + { + value: number; + unit: string; + } + > & { + "score.total": { + value: number; + unit: string; + }; + "score.fcp": { + value: number; + unit: string; + }; + "score.lcp": { + value: number; + unit: string; + }; + "score.fid": { + value: number; + unit: string; + }; + "score.cls": { + value: number; + unit: string; + }; + "score.ttfb": { + value: number; + unit: string; + }; + "score.weight.fcp": { + value: number; + unit: string; + }; + "score.weight.lcp": { + value: number; + unit: string; + }; + "score.weight.fid": { + value: number; + unit: string; + }; + "score.weight.cls": { + value: number; + unit: string; + }; + "score.weight.ttfb": { + value: number; + unit: string; + }; + }; +}; + +export type MetricScoreProps = { + fcpScore: number; + lcpScore: number; + clsScore: number; + fidScore: number; + ttfbScore: number; +}; +export type MetricWeightsProps = { + fcp: number; + lcp: number; + cls: number; + fid: number; + ttfb: number; +}; + +export type AIToolCall = { + toolCallId: string; + toolName: string; + args: Record; + result?: Record | string; + state?: string; + step?: number; +}; + +// Content can be string (v1) or array of content blocks (v2) +export type AIContentBlock = { + type: string; + text?: string; +}; + +export type AIMessageContent = string | AIContentBlock[]; + +export type AIMessage = { + role: string; + content: AIMessageContent; + toolInvocations?: AIToolCall[]; + parts?: unknown[]; +}; + +export type AIPrompt = { + system?: string; + messages?: AIMessage[]; +}; + +export type AIResponse = { + finishReason?: string; + text?: string; + toolCalls?: AIToolCall[]; +}; + +export type AIMetadata = { + modelId?: string; + modelProvider?: string; + functionId?: string; + metadata: Record; + maxRetries?: number; + maxSteps?: number; + promptTokens?: number; + completionTokens?: number; +}; + +export type AITrace = { + id: string; // root span_id, also used as traceId for display + name: string; // root span description or op for primary display + operation: string; + timestamp: number; + durationMs: number; + tokensDisplay: string; // prompt/completion or N/A + promptTokens?: number; + completionTokens?: number; + hasToolCall: boolean; + rawSpan: Span; + metadata: AIMetadata; + prompt?: AIPrompt; + response?: AIResponse; + toolCalls: AIToolCall[]; +}; + +export type AILibraryHandler = { + id: string; + name: string; + canHandleSpan: (span: Span) => boolean; + extractRootSpans: (spans: Span[]) => Span[]; + processTrace: (rootSpan: Span) => AITrace; + getDisplayTitle: (trace: AITrace) => string; + getTypeBadge: (trace: AITrace) => string; + getTokensDisplay: (trace: AITrace) => string; +}; diff --git a/packages/spotlight/src/telemetry/utils/profileUtils.ts b/packages/spotlight/src/telemetry/utils/profileUtils.ts new file mode 100644 index 000000000..4f290c05c --- /dev/null +++ b/packages/spotlight/src/telemetry/utils/profileUtils.ts @@ -0,0 +1,15 @@ +import type { EventFrame } from "../types.ts"; + +const _FUNCTION_NAME_FROM_FRAME_CACHE = new WeakMap(); +export function getFunctionNameFromFrame(frame: EventFrame): string { + let result = _FUNCTION_NAME_FROM_FRAME_CACHE.get(frame); + if (!result) { + const module = frame.module || frame.filename || frame.abs_path || ""; + const functionName = frame.function || ""; + const lineNo = frame.lineno ? `:${frame.lineno}` : ""; + const colNo = frame.lineno && frame.colno ? `:${frame.colno}` : ""; + result = `${module}@${functionName}${lineNo}${colNo}`; + _FUNCTION_NAME_FROM_FRAME_CACHE.set(frame, result); + } + return result; +} diff --git a/packages/spotlight/src/telemetry/utils/sdkToPlatform.ts b/packages/spotlight/src/telemetry/utils/sdkToPlatform.ts new file mode 100644 index 000000000..e7cd4215f --- /dev/null +++ b/packages/spotlight/src/telemetry/utils/sdkToPlatform.ts @@ -0,0 +1,15 @@ +/** + * Normalizes SDK name to a platform. Can be as specific or unspecific as we support different platforms and SDKs. + */ +export function sdkToPlatform(name: string) { + if (name.includes("javascript.astro")) return "javascript.astro"; + if (name.includes("javascript")) return "javascript"; + if (name.includes("java")) return "java"; + if (name.includes("python")) return "python"; + if (name.includes("php.laravel")) return "php.laravel"; + if (name.includes("php.symfony")) return "php.symfony"; + if (name.includes("php")) return "php"; + if (name.includes("ruby")) return "ruby"; + if (name.includes("dotnet")) return "dotnet"; + return "unknown"; +} diff --git a/packages/spotlight/src/telemetry/utils/sentry.ts b/packages/spotlight/src/telemetry/utils/sentry.ts new file mode 100644 index 000000000..01d0466ca --- /dev/null +++ b/packages/spotlight/src/telemetry/utils/sentry.ts @@ -0,0 +1,42 @@ +import { + ERROR_EVENT_TYPES, + FEEDBACK_EVENT_TYPES, + LOG_EVENT_TYPES, + PROFILE_EVENT_TYPES, + TRACE_EVENT_TYPES, +} from "../constants/sentry.ts"; +import type { + SentryErrorEvent, + SentryEvent, + SentryFeedbackEvent, + SentryLogEvent, + SentryProfileV1Event, + SentryProfileV2ChunkEvent, + SentryTransactionEvent, +} from "../types.ts"; + +export function isErrorEvent(event: SentryEvent): event is SentryErrorEvent { + return (!event.type || ERROR_EVENT_TYPES.has(event.type)) && Boolean((event as SentryErrorEvent).exception); +} + +export function isProfileEvent(event: SentryEvent): event is SentryProfileV1Event { + return !!event.type && PROFILE_EVENT_TYPES.has(event.type) && (event as SentryProfileV1Event).version === "1"; +} + +export function isProfileChunkEvent(event: SentryEvent): event is SentryProfileV2ChunkEvent { + return event.type === "profile_chunk" && (event as SentryProfileV2ChunkEvent).version === "2"; +} + +export function isTraceEvent(event: SentryEvent): event is SentryTransactionEvent { + return !!event.type && TRACE_EVENT_TYPES.has(event.type); +} + +export function isLogEvent(event: SentryEvent): event is SentryLogEvent { + return !!event.type && LOG_EVENT_TYPES.has(event.type); +} + +export function isFeedbackEvent(event: SentryEvent): event is SentryFeedbackEvent { + return ( + !!event.type && FEEDBACK_EVENT_TYPES.has(event.type) && Boolean((event as SentryFeedbackEvent).contexts?.feedback) + ); +} diff --git a/packages/spotlight/src/telemetry/utils/traces.ts b/packages/spotlight/src/telemetry/utils/traces.ts new file mode 100644 index 000000000..cab6dfd96 --- /dev/null +++ b/packages/spotlight/src/telemetry/utils/traces.ts @@ -0,0 +1,87 @@ +import { log } from "../logger.ts"; +import type { Span, Trace } from "../types.ts"; + +// mutates spans in place and adds children, as well as returns the top level tree +export function groupSpans(spans: Map): Span[] { + const tree: Span[] = []; + + // need to sort root(s) first + const sortedSpans = Array.from(spans.values()).sort((a, b) => { + const parentComp = (a.parent_span_id ? 1 : 0) - (b.parent_span_id ? 1 : 0); + return parentComp === 0 ? compareSpans(a, b) : parentComp; + }); + + for (const span of sortedSpans) { + let parent = span && getParentOfSpan(span, spans, sortedSpans); + + span.children ??= []; + if (parent) { + parent.children ??= []; + parent.children.push(span); + } else if (span.parent_span_id) { + const parentParent = sortedSpans.find(s => !s.parent_span_id); + if (!parentParent) { + log(`Root span (${span.parent_span_id}) for span (${span.span_id}). Creating orphan.`); + } else { + log(`Creating orphan for parent (${span.parent_span_id}) for span (${span.span_id})`); + } + parent = { + trace_id: span.trace_id, + span_id: span.parent_span_id, + parent_span_id: parentParent ? parentParent.span_id : null, + op: "orphan", + description: "missing or unknown parent span", + children: [span], + start_timestamp: span.start_timestamp, + timestamp: span.timestamp, + status: "unknown", + }; + spans.set(parent.span_id, parent); + // sortedSpans.splice(spanIdx, 0, parent); + if (parentParent) { + parentParent.children ??= []; + parentParent.children.push(parent); + } else { + tree.push(parent); + } + } else { + tree.push(span); + } + spans.set(span.span_id, span); + } + + return tree; +} + +function getParentOfSpan(span: Span, idLookup: Map, allSpans: Span[]): Span | undefined { + if (!span.parent_span_id) { + return undefined; + } + + return idLookup.get(span.parent_span_id) || allSpans.find(s => s.span_id === span.parent_span_id); +} + +export function compareSpans(a: { start_timestamp: number }, b: { start_timestamp: number }): number { + return a.start_timestamp - b.start_timestamp; +} + +export function getRootTransactionMethod(trace: Trace) { + // Fallback to first transaction if no root transaction (e.g., orphan traces) + const transaction = trace.rootTransaction || trace.transactions[0]; + const method = String(transaction?.contexts?.trace.data?.method || transaction?.request?.method || ""); + return method; +} + +export function getRootTransactionName(trace: Trace) { + const method = getRootTransactionMethod(trace); + // For orphan traces without a root, use the first transaction's name + let name = trace.rootTransactionName; + if (!trace.rootTransaction && trace.transactions.length > 0) { + name = trace.transactions[0].transaction || trace.rootTransactionName; + } + + // Strip method prefix if present + const finalName = method && name.startsWith(method) ? name.slice(method.length + 1) : name; + + return finalName; +} diff --git a/packages/spotlight/src/telemetry/utils/webVitals.ts b/packages/spotlight/src/telemetry/utils/webVitals.ts new file mode 100644 index 000000000..b3d93434f --- /dev/null +++ b/packages/spotlight/src/telemetry/utils/webVitals.ts @@ -0,0 +1,167 @@ +import type { SentryEvent } from "../types.ts"; + +const SQRT_2 = Math.sqrt(2); + +// Gauss error function +function erf(x: number): number { + const a1 = 0.254829592; + const a2 = -0.284496736; + const a3 = 1.421413741; + const a4 = -1.453152027; + const a5 = 1.061405429; + const p = 0.3275911; + + const sign = x < 0 ? -1 : 1; + const absX = Math.abs(x); + + const t = 1.0 / (1.0 + p * absX); + const y = 1.0 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX); + + return sign * y; +} + +// Sigma function for CDF score calculation +export function calculateCdfSigma(p10: number, p50: number): number { + return Math.abs(Math.log(p10) - Math.log(p50)) / (SQRT_2 * 0.9061938024368232); +} + +// Calculates a log-normal CDF score based on a log-normal with a specific p10 and p50 +export function calculateCdfScore(value: number, p10: number, p50: number): number { + return 0.5 * (1.0 - erf((Math.log(value) - Math.log(p50)) / (SQRT_2 * calculateCdfSigma(p50, p10)))); +} + +interface ScoreComponent { + measurement: string; + weight: number; + p10: number; + p50: number; + optional?: boolean; +} + +interface Profile { + name: string; + scoreComponents: ScoreComponent[]; + condition?: { + op: string; + name: string; + value: string | number; + }; +} + +export interface PerformanceScoreConfig { + profiles: Profile[]; +} + +// export function getPropertyByPath(obj: Record, path: string): any { +// const keys = path.split('.'); +// let result = obj; + +// for (const key of keys) { +// if (result && result[key]) { +// result = result[key]; +// } else { +// return undefined; +// } +// } + +// return result; +// } + +// export function matchCondition(condition: { op: string; name: string; value: string | number }, event: SentryEvent): boolean { +// const { op, name, value } = condition; + +// if (op === 'eq') { +// const eventValue = getPropertyByPath({ event }, name); +// return eventValue === value; +// } + +// return false; +// } + +export function normalizePerformanceScore( + event: SentryEvent, + performanceScore: PerformanceScoreConfig | undefined, +): void { + if (!performanceScore || event.type === "profile_chunk") { + return; + } + + for (const profile of performanceScore.profiles) { + // TODO: To check a matching condition as done in relay. Skipped right now because Browser type is not coming in Event data + // const condition = profile.condition; + // if (condition && !matchCondition(condition, event)) { + // continue; + // } + + const measurements = event.measurements; + + if (measurements) { + let shouldAddTotal = false; + + // TODO: To calculate performance score if all required metric fields are present in event measurement data. + // if ( + // profile.scoreComponents.some(c => { + // return ( + // !Object.prototype.hasOwnProperty.call(measurements, c.measurement) && + // Math.abs(c.weight) >= Number.EPSILON && + // !c.optional + // ); + // }) + // ) { + // break; + // } + + let scoreTotal = 0.0; + let weightTotal = 0.0; + + for (const component of profile.scoreComponents) { + if (component.optional && !Object.prototype.hasOwnProperty.call(measurements, component.measurement)) { + continue; + } + + weightTotal += component.weight; + } + + if (Math.abs(weightTotal) < Number.EPSILON) { + break; + } + + for (const component of profile.scoreComponents) { + let normalizedComponentWeight = 0.0; + + if (Object.prototype.hasOwnProperty.call(measurements, component.measurement)) { + normalizedComponentWeight = component.weight / weightTotal; + const value = measurements[component.measurement].value; + const cdf = calculateCdfScore(Math.max(0.0, value), component.p10, component.p50); + const componentScore = cdf * normalizedComponentWeight; + scoreTotal += componentScore; + shouldAddTotal = true; + + measurements[`score.${component.measurement}`] = { + value: cdf, + unit: "ratio", + }; + } else { + measurements[`score.${component.measurement}`] = { + value: 0, + unit: "ratio", + }; + } + + measurements[`score.weight.${component.measurement}`] = { + value: normalizedComponentWeight, + unit: "ratio", + }; + } + + if (shouldAddTotal && !Object.prototype.hasOwnProperty.call(measurements, "score.total")) { + measurements["score.total"] = { + value: scoreTotal, + unit: "ratio", + }; + } + + break; + } + } +} diff --git a/packages/spotlight/src/telemetry/uuid.ts b/packages/spotlight/src/telemetry/uuid.ts new file mode 100644 index 000000000..1b7c4bff6 --- /dev/null +++ b/packages/spotlight/src/telemetry/uuid.ts @@ -0,0 +1,9 @@ +export function generateUuidv4() { + let dt = new Date().getTime(); + return "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, c => { + let rnd = Math.random() * 16; //random number in range 0 to 16 + rnd = ((dt + rnd) % 16) | 0; + dt = Math.floor(dt / 16); + return (c === "x" ? rnd : (rnd & 0x3) | 0x8).toString(16); + }); +} diff --git a/packages/spotlight/src/ui/lib/logger.ts b/packages/spotlight/src/ui/lib/logger.ts index 14f7cd1ce..8e1436ef6 100644 --- a/packages/spotlight/src/ui/lib/logger.ts +++ b/packages/spotlight/src/ui/lib/logger.ts @@ -1,23 +1 @@ -export const SPOTLIGHT_PREFIX = "🔎 [Spotlight]"; - -const noop = (..._args: unknown[]) => {}; // eslint-disable-line @typescript-eslint/no-unused-vars -let _log = noop; -let _warn = noop; - -export function activateLogger() { - _log = (...args: unknown[]) => console.log(SPOTLIGHT_PREFIX, ...args); - _warn = (...args: unknown[]) => console.warn(SPOTLIGHT_PREFIX, ...args); -} - -export function deactivateLogger() { - _log = noop; - _warn = noop; -} - -export function log(...args: unknown[]) { - _log(...args); -} - -export function warn(...args: unknown[]) { - _warn(...args); -} +export * from "../../telemetry/logger.ts"; diff --git a/packages/spotlight/src/ui/lib/uuid.ts b/packages/spotlight/src/ui/lib/uuid.ts index 1b7c4bff6..e942e7dc7 100644 --- a/packages/spotlight/src/ui/lib/uuid.ts +++ b/packages/spotlight/src/ui/lib/uuid.ts @@ -1,9 +1 @@ -export function generateUuidv4() { - let dt = new Date().getTime(); - return "xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, c => { - let rnd = Math.random() * 16; //random number in range 0 to 16 - rnd = ((dt + rnd) % 16) | 0; - dt = Math.floor(dt / 16); - return (c === "x" ? rnd : (rnd & 0x3) | 0x8).toString(16); - }); -} +export * from "../../telemetry/uuid.ts"; diff --git a/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx b/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx new file mode 100644 index 000000000..684ee46bb --- /dev/null +++ b/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx @@ -0,0 +1,31 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, expect, test } from "vitest"; +import { createTelemetry } from "../../../../telemetry/index.ts"; +import EventContexts from "./EventContexts"; + +afterEach(cleanup); + +test("renders modules alongside extra data without mutating the SDK event", async () => { + const telemetry = createTelemetry(); + await telemetry.ingestEnvelope([ + {}, + [ + [ + { type: "event" }, + { + event_id: "context-event", + timestamp: 1, + extra: { environment: "local" }, + modules: { react: "19.1" }, + }, + ], + ], + ]); + const event = telemetry.getEvent("context-event")!; + expect(Object.isFrozen(event.extra)).toBe(true); + render(); + expect(screen.getByText('"local"')).toBeDefined(); + expect(screen.getByText('"19.1"')).toBeDefined(); + expect(event.extra).toEqual({ environment: "local" }); + telemetry.dispose(); +}); diff --git a/packages/spotlight/src/ui/telemetry/components/events/EventContexts.tsx b/packages/spotlight/src/ui/telemetry/components/events/EventContexts.tsx index 7552d18cc..842eccec4 100644 --- a/packages/spotlight/src/ui/telemetry/components/events/EventContexts.tsx +++ b/packages/spotlight/src/ui/telemetry/components/events/EventContexts.tsx @@ -1,4 +1,4 @@ -import type { SentryEvent } from "../../types"; +import type { ProcessedEvent } from "../../../../sdk/core.ts"; import Attachment from "../insights/envelopes/Attachment"; import { CodeViewer } from "../insights/envelopes/CodeViewer"; import { ContextView } from "../shared/ContextView"; @@ -18,12 +18,12 @@ const exampleContext = ( ); -export default function EventContexts({ event }: { event: SentryEvent }) { +export default function EventContexts({ event }: { event: ProcessedEvent }) { if (!event) { return exampleContext; } - const contexts: Record | null> = { + const contexts: Record | null | undefined> = { request: event.request, ...event.contexts, }; @@ -33,7 +33,7 @@ export default function EventContexts({ event }: { event: SentryEvent }) { } if (event.modules) { - contexts.extra = Object.assign(contexts.extra || {}, { modules: event.modules }); + contexts.extra = { ...contexts.extra, modules: event.modules }; } const contextEntries = Object.entries(contexts).filter(entry => entry[1]) as [string, Record][]; diff --git a/packages/spotlight/src/ui/telemetry/components/insights/Queries.tsx b/packages/spotlight/src/ui/telemetry/components/insights/Queries.tsx index 11be65a7d..384718f80 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/Queries.tsx +++ b/packages/spotlight/src/ui/telemetry/components/insights/Queries.tsx @@ -2,21 +2,17 @@ import { ReactComponent as Sort } from "@spotlight/ui/assets/sort.svg"; import { ReactComponent as SortDown } from "@spotlight/ui/assets/sortDown.svg"; import { cn } from "@spotlight/ui/lib/cn"; import { TimeBar } from "@spotlight/ui/telemetry/components/shared/TimeBar"; -import { DB_SPAN_REGEX, QUERIES_HEADERS, QUERIES_SORT_KEYS } from "@spotlight/ui/telemetry/constants"; +import { QUERIES_HEADERS, QUERIES_SORT_KEYS } from "@spotlight/ui/telemetry/constants"; import { useSentrySpans } from "@spotlight/ui/telemetry/data/useSentrySpans"; import useSort from "@spotlight/ui/telemetry/hooks/useSort"; -import type { Span } from "@spotlight/ui/telemetry/types"; import { getFormattedDuration, getSpanDurationClassName } from "@spotlight/ui/telemetry/utils/duration"; import Table from "@spotlight/ui/ui/table"; import { useMemo } from "react"; import { useNavigate } from "react-router-dom"; +import { type QuerySummary, getQuerySummaries } from "../../../../telemetry/selectors/database.ts"; import EmptyState from "../shared/EmptyState"; -type QueryInfo = { - avgDuration: number; - totalTime: number; - description: string; -}; +type QueryInfo = QuerySummary; type QueryInfoComparator = (a: QueryInfo, b: QueryInfo) => number; type QuerySortTypes = (typeof QUERIES_SORT_KEYS)[keyof typeof QUERIES_SORT_KEYS]; @@ -26,21 +22,8 @@ const COMPARATORS: Record = { if (a.description > b.description) return 1; return 0; }, - [QUERIES_SORT_KEYS.avgDuration]: (a, b) => a.avgDuration - b.avgDuration, - [QUERIES_SORT_KEYS.totalTime]: (a, b) => a.totalTime - b.totalTime, -}; - -const calculateQueryInfo = ({ query, spanData }: { query: string; spanData: Span[] }): QueryInfo => { - const queryTypeData = spanData.filter((span: Span) => span.description === query); - const times = queryTypeData.map((span: Span) => span.timestamp - span.start_timestamp); - const totalTimeInMs = times.reduce((acc: number, time: number) => acc + time, 0); - const avgDuration = totalTimeInMs / times.length; - - return { - avgDuration, - totalTime: totalTimeInMs, - description: query, - }; + [QUERIES_SORT_KEYS.avgDuration]: (a, b) => a.averageDurationMs - b.averageDurationMs, + [QUERIES_SORT_KEYS.totalTime]: (a, b) => a.totalDurationMs - b.totalDurationMs, }; const Queries = () => { @@ -50,17 +33,10 @@ const Queries = () => { const queriesData: QueryInfo[] = useMemo(() => { const compareQueryInfo = COMPARATORS[sort.active] || COMPARATORS[QUERIES_SORT_KEYS.totalTime]; - const spans = allSpans; - const onlyDBSpans = spans.filter((span: Span) => DB_SPAN_REGEX.test(span.op || "")); - const uniqueSpansSet = new Set(onlyDBSpans.map(span => (span.description ?? "").trim())); - // Clear out empty ones (they collapse as a single empty string since this is a set) - uniqueSpansSet.delete(""); - return [...uniqueSpansSet] - .map(query => calculateQueryInfo({ query, spanData: onlyDBSpans })) - .sort((a, b) => (sort.asc ? compareQueryInfo(a, b) : compareQueryInfo(b, a))); + return getQuerySummaries(allSpans).sort((a, b) => (sort.asc ? compareQueryInfo(a, b) : compareQueryInfo(b, a))); }, [allSpans, sort]); - const maxTime = Math.max(...queriesData.map(query => query.totalTime)); + const maxTime = Math.max(...queriesData.map(query => query.totalDurationMs)); const handleRowClick = (query: QueryInfo) => { navigate(`/telemetry/insights/queries/${btoa(query.description)}`); @@ -128,16 +104,23 @@ const Queries = () => { className="hover:bg-primary-900 cursor-pointer" > - + {query.description} - {getFormattedDuration(query.totalTime)} + + {getFormattedDuration(query.totalDurationMs)} + - - {getFormattedDuration(query.avgDuration)} + + {getFormattedDuration(query.averageDurationMs)} diff --git a/packages/spotlight/src/ui/telemetry/components/insights/QuerySummary.tsx b/packages/spotlight/src/ui/telemetry/components/insights/QuerySummary.tsx index b36667795..88c896b81 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/QuerySummary.tsx +++ b/packages/spotlight/src/ui/telemetry/components/insights/QuerySummary.tsx @@ -6,6 +6,7 @@ import Breadcrumbs from "@spotlight/ui/ui/breadcrumbs"; import Table from "@spotlight/ui/ui/table"; import { useMemo } from "react"; import { Link, useParams } from "react-router-dom"; +import { getQuerySummaries } from "../../../../telemetry/selectors/database.ts"; import { QUERY_SUMMARY_HEADERS, QUERY_SUMMARY_SORT_KEYS } from "../../constants"; import { useSentrySpans } from "../../data/useSentrySpans"; import useSort from "../../hooks/useSort"; @@ -48,9 +49,9 @@ const QuerySummary = () => { const spans = allSpans; const compareSpanInfo = COMPARATORS[sort.active] || COMPARATORS[QUERY_SUMMARY_SORT_KEYS.totalTime]; - return spans - .filter(span => span.description === decodedType) - .sort((a, b) => (sort.asc ? compareSpanInfo(a, b) : compareSpanInfo(b, a))); + return (getQuerySummaries(spans, { description: decodedType })[0]?.spans ?? []).sort((a, b) => + sort.asc ? compareSpanInfo(a, b) : compareSpanInfo(b, a), + ); }, [allSpans, sort, decodedType]); if (!filteredDBSpans || !filteredDBSpans.length) { diff --git a/packages/spotlight/src/ui/telemetry/components/insights/Resources.tsx b/packages/spotlight/src/ui/telemetry/components/insights/Resources.tsx index c1fbe8e7c..5800b462c 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/Resources.tsx +++ b/packages/spotlight/src/ui/telemetry/components/insights/Resources.tsx @@ -3,57 +3,15 @@ import { ReactComponent as SortDown } from "@spotlight/ui/assets/sortDown.svg"; import { cn } from "@spotlight/ui/lib/cn"; import Table from "@spotlight/ui/ui/table"; import { useMemo } from "react"; +import { type ResourceSummary, getResourceSummaries } from "../../../../telemetry/selectors/resources.ts"; import { RESOURCES_SORT_KEYS, RESOURCE_HEADERS } from "../../constants"; import { useSentrySpans } from "../../data/useSentrySpans"; import useSort from "../../hooks/useSort"; -import type { Span } from "../../types"; import { formatBytes } from "../../utils/bytes"; import { getFormattedDuration, getSpanDurationClassName } from "../../utils/duration"; import EmptyState from "../shared/EmptyState"; -type ResourceInfo = { - avgDuration: number; - totalTime: number; - description: string; - avgEncodedSize: number; - similarResources: Span[]; -}; - -const calculateResourceInfo = ({ resource, spanData }: { resource: string; spanData: Span[] }): ResourceInfo => { - const specificResources = spanData.filter((span: Span) => span.description === resource); - const totalTimeInMs = specificResources.reduce( - (acc: number, span: Span) => acc + (span.timestamp - span.start_timestamp), - 0, - ); - const avgDuration = totalTimeInMs / specificResources.length; - const avgEncodedSize = - specificResources.reduce((acc: number, span: Span) => { - const contentLength = span.data?.["http.response_content_length"]; - if (typeof contentLength === "number") { - return acc + contentLength; - } - return acc; - }, 0) / specificResources.length; - - return { - avgDuration, - totalTime: totalTimeInMs, - description: resource, - avgEncodedSize, - similarResources: specificResources, - }; -}; - -const getResourceSpans = (spans: Span[], options: { type?: string; regex?: RegExp }) => { - if (options.type) { - return spans.filter((span: Span) => span.description === options.type); - } - if (options.regex) { - const regex = new RegExp(options.regex); - return spans.filter((span: Span) => regex.test(span.op || "")); - } - return []; -}; +type ResourceInfo = ResourceSummary; type ResourceInfoComparator = (a: ResourceInfo, b: ResourceInfo) => number; type ResourceSortTypes = (typeof RESOURCES_SORT_KEYS)[keyof typeof RESOURCES_SORT_KEYS]; @@ -63,9 +21,9 @@ const COMPARATORS: Record = { if (a.description > b.description) return 1; return 0; }, - [RESOURCES_SORT_KEYS.avgEncodedSize]: (a, b) => a.avgEncodedSize - b.avgEncodedSize, - [RESOURCES_SORT_KEYS.avgDuration]: (a, b) => a.avgDuration - b.avgDuration, - [RESOURCES_SORT_KEYS.totalTime]: (a, b) => a.totalTime - b.totalTime, + [RESOURCES_SORT_KEYS.avgEncodedSize]: (a, b) => a.averageEncodedSizeBytes - b.averageEncodedSizeBytes, + [RESOURCES_SORT_KEYS.avgDuration]: (a, b) => a.averageDurationMs - b.averageDurationMs, + [RESOURCES_SORT_KEYS.totalTime]: (a, b) => a.totalDurationMs - b.totalDurationMs, }; const Resources = () => { @@ -73,18 +31,10 @@ const Resources = () => { const { sort, toggleSortOrder } = useSort({ defaultSortType: RESOURCES_SORT_KEYS.totalTime }); const resources = useMemo(() => { - const filteredResourceSpans = getResourceSpans(allSpans, { regex: /resource\.[A-Za-z]+/ }); - const uniqueResourceDescriptionsSet = new Set(filteredResourceSpans.map(span => (span.description ?? "").trim())); - // Clear out empty ones (they collapse as a single empty string since this is a set) - uniqueResourceDescriptionsSet.delete(""); - const uniqueResourceDescriptions: string[] = [...uniqueResourceDescriptionsSet]; const compareResourceInfo = COMPARATORS[sort.active] || COMPARATORS[RESOURCES_SORT_KEYS.totalTime]; - - return uniqueResourceDescriptions - .map(resource => calculateResourceInfo({ resource, spanData: filteredResourceSpans })) - .sort((a, b) => { - return sort.asc ? compareResourceInfo(a, b) : compareResourceInfo(b, a); - }); + return getResourceSummaries(allSpans).sort((a, b) => + sort.asc ? compareResourceInfo(a, b) : compareResourceInfo(b, a), + ); }, [sort, allSpans]); if (!resources?.length) { @@ -143,17 +93,17 @@ const Resources = () => { {resource.description} - - {getFormattedDuration(resource.avgDuration)} + + {getFormattedDuration(resource.averageDurationMs)} - - {getFormattedDuration(resource.totalTime)} + + {getFormattedDuration(resource.totalDurationMs)} - {formatBytes(resource.avgEncodedSize)} + {formatBytes(resource.averageEncodedSizeBytes)} ))} diff --git a/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/aiLibraries.ts b/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/aiLibraries.ts index 3591894ae..4da3f6826 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/aiLibraries.ts +++ b/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/aiLibraries.ts @@ -1,60 +1 @@ -import type { AILibraryHandler, Span, SpotlightAITrace, Trace } from "@spotlight/ui/telemetry/types"; -import { genAIHandler } from "./genAI"; -import { vercelAISDKHandler } from "./vercelAISDK"; - -// Registry of AI span handlers. The generic gen_ai handler runs first because -// it only claims standard `gen_ai.*` spans that lack `vercel.ai.*` fields, so -// the Vercel handler still picks up its own proprietary spans. -const aiLibraries: AILibraryHandler[] = [genAIHandler, vercelAISDKHandler]; - -export { aiLibraries }; - -export function detectAILibraryHandler(span: Span): AILibraryHandler | null { - for (const handler of aiLibraries) { - // careful if we support multiple libraries, we should improve this method so it doesn't return the first one - if (handler.canHandleSpan(span)) { - return handler; - } - } - return null; -} - -export function extractAllAIRootSpans(spans: Span[]): { span: Span; handler: AILibraryHandler }[] { - const results: { span: Span; handler: AILibraryHandler }[] = []; - // Handlers can match overlapping span shapes (both AI handlers key off the - // `gen_ai.` op prefix). Attribute each root span to the first handler that - // claims it so a span is never surfaced twice. - const claimed = new Set(); - - for (const handler of aiLibraries) { - const rootSpans = handler.extractRootSpans(spans); - for (const rootSpan of rootSpans) { - if (claimed.has(rootSpan.span_id)) { - continue; - } - claimed.add(rootSpan.span_id); - results.push({ span: rootSpan, handler }); - } - } - - return results; -} - -export function createAITraceFromSpan(span: Span): SpotlightAITrace | null { - const handler = detectAILibraryHandler(span); - if (!handler) { - return null; - } - - return handler.processTrace(span); -} - -export function hasAISpans(trace: Trace): boolean { - if (!trace.spanTree) { - return false; - } - - // TODO: We may want to cache/optimize this `extractAllAIRootSpans` helper - const aiRootSpans = extractAllAIRootSpans(trace.spanTree); - return aiRootSpans.length > 0; -} +export * from "../../../../../../telemetry/ai/aiLibraries.ts"; diff --git a/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/genAI.ts b/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/genAI.ts index 4b0f8565b..3fd67b5d7 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/genAI.ts +++ b/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/genAI.ts @@ -1,357 +1 @@ -import { getAllSpansInTree } from "@spotlight/ui/telemetry/store/helpers"; -import type { AILibraryHandler, AIToolCall, Span, SpotlightAITrace } from "@spotlight/ui/telemetry/types"; - -// Generic handler for the standardized `gen_ai.*` span conventions. Any SDK -// that follows sentry-conventions (sentry-python's PydanticAI/OpenAI/Anthropic -// integrations, the OTel GenAI conventions, ...) maps through here regardless -// of provider — the provider is just the `gen_ai.system` attribute, not a -// reason to fork the handler. The Vercel AI SDK keeps its own handler only -// because it emits proprietary `vercel.ai.*` fields on top of `gen_ai.*`. -// https://github.com/getsentry/sentry-conventions/tree/main/model/attributes/gen_ai -const GEN_AI_OP_PREFIX = "gen_ai."; -const VERCEL_FIELD_PREFIX = "vercel.ai."; - -// Span ops -const GEN_AI_CHAT_OP = "gen_ai.chat"; -const GEN_AI_INVOKE_AGENT_OP = "gen_ai.invoke_agent"; -const GEN_AI_EXECUTE_TOOL_OP = "gen_ai.execute_tool"; - -// Span data fields (sentry-conventions gen_ai.*) -const GEN_AI_OPERATION_NAME_FIELD = "gen_ai.operation.name"; -const GEN_AI_AGENT_NAME_FIELD = "gen_ai.agent.name"; -const GEN_AI_SYSTEM_FIELD = "gen_ai.system"; -const GEN_AI_REQUEST_MODEL_FIELD = "gen_ai.request.model"; -const GEN_AI_RESPONSE_MODEL_FIELD = "gen_ai.response.model"; -const GEN_AI_USAGE_INPUT_TOKENS_FIELD = "gen_ai.usage.input_tokens"; -const GEN_AI_USAGE_OUTPUT_TOKENS_FIELD = "gen_ai.usage.output_tokens"; -const GEN_AI_REQUEST_MESSAGES_FIELD = "gen_ai.request.messages"; -const GEN_AI_SYSTEM_INSTRUCTIONS_FIELD = "gen_ai.system_instructions"; -const GEN_AI_RESPONSE_TEXT_FIELD = "gen_ai.response.text"; -const GEN_AI_RESPONSE_TOOL_CALLS_FIELD = "gen_ai.response.tool_calls"; -const GEN_AI_RESPONSE_FINISH_REASONS_FIELD = "gen_ai.response.finish_reasons"; -const GEN_AI_TOOL_NAME_FIELD = "gen_ai.tool.name"; -const GEN_AI_TOOL_CALL_ID_FIELD = "gen_ai.tool.call.id"; -const GEN_AI_TOOL_INPUT_FIELD = "gen_ai.tool.input"; -const GEN_AI_TOOL_OUTPUT_FIELD = "gen_ai.tool.output"; - -const DEFAULT_TRACE_NAME = "AI Interaction"; -const UNKNOWN_OPERATION = "N/A"; - -const OPERATION_BADGE_MAP = new Map([ - ["chat", "Chat"], - ["invoke_agent", "Invoke Agent"], - ["execute_tool", "Execute Tool"], - [GEN_AI_CHAT_OP, "Chat"], - [GEN_AI_INVOKE_AGENT_OP, "Invoke Agent"], - [GEN_AI_EXECUTE_TOOL_OP, "Execute Tool"], -]); - -function isGenAISpan(span: Span): boolean { - return !!span.op?.toLowerCase().startsWith(GEN_AI_OP_PREFIX); -} - -/** - * Claims any standard `gen_ai.*` span that isn't a Vercel AI SDK span. Both - * carry the `gen_ai.` op prefix, but only the Vercel SDK sets `vercel.ai.*` - * data fields, so we defer those to the Vercel handler. A generic gen_ai span - * has at least one `gen_ai.*` data key and no `vercel.ai.*` key. - */ -function isGenAIConventionSpan(span: Span): boolean { - if (!isGenAISpan(span) || !span.data) { - return false; - } - - let hasGenAIField = false; - for (const key of Object.keys(span.data)) { - if (key.startsWith(VERCEL_FIELD_PREFIX)) { - return false; - } - if (key.startsWith(GEN_AI_OP_PREFIX)) { - hasGenAIField = true; - } - } - - return hasGenAIField; -} - -export const genAIHandler: AILibraryHandler = { - id: "gen-ai", - name: "Gen AI", - - canHandleSpan: isGenAIConventionSpan, - - extractRootSpans: (spans: Span[]): Span[] => { - const resultRoots: Span[] = []; - - const findAndCollectAIRoots = (spansToSearch: Span[]) => { - for (const currentSpan of spansToSearch) { - if (isGenAIConventionSpan(currentSpan)) { - resultRoots.push(currentSpan); - } else if (!isGenAISpan(currentSpan) && currentSpan.children?.length) { - // Stop at any gen_ai.* span: it's the shallowest AI root for its - // tree (claimed by this handler or the Vercel one), so its nested - // gen_ai.* children must not be surfaced as separate roots. - findAndCollectAIRoots(currentSpan.children); - } - } - }; - - findAndCollectAIRoots(spans); - return resultRoots; - }, - - processTrace: (rootSpan: Span): SpotlightAITrace => { - const allSpans = getAllSpansInTree(rootSpan); - const tokenUsage = extractTokenUsage(allSpans); - const operation = determineOperation(rootSpan); - const hasToolCall = allSpans.some( - span => span.op === GEN_AI_EXECUTE_TOOL_OP || !!span.data?.[GEN_AI_TOOL_NAME_FIELD], - ); - - const durationMs = rootSpan.timestamp - rootSpan.start_timestamp; - - const trace: SpotlightAITrace = { - id: rootSpan.span_id, - name: determineTraceName(rootSpan), - operation, - timestamp: rootSpan.start_timestamp, - durationMs, - tokensDisplay: formatTokensDisplay(tokenUsage.promptTokens, tokenUsage.completionTokens), - promptTokens: tokenUsage.promptTokens, - completionTokens: tokenUsage.completionTokens, - hasToolCall, - rawSpan: rootSpan, - metadata: { - metadata: {}, - promptTokens: tokenUsage.promptTokens, - completionTokens: tokenUsage.completionTokens, - }, - toolCalls: [], - }; - - const responseState = { finishReasonFromText: false }; - for (const span of allSpans) { - if (!span.data) continue; - extractMetadata(span, trace); - extractPromptData(span, trace); - extractResponseData(span, trace, responseState); - extractToolCallData(span, trace); - } - - return trace; - }, - - getDisplayTitle: (trace: SpotlightAITrace): string => { - return trace.name || trace.operation || `AI Trace ${trace.id.substring(0, 8)}`; - }, - - getTypeBadge: (trace: SpotlightAITrace): string => { - if (trace.hasToolCall) { - return "Tool-Call"; - } - - return OPERATION_BADGE_MAP.get(trace.operation) ?? trace.operation.replace(/^gen_ai\./, ""); - }, - - getTokensDisplay: (trace: SpotlightAITrace): string => { - return trace.tokensDisplay; - }, -}; - -function determineOperation(rootSpan: Span): string { - const operationName = rootSpan.data?.[GEN_AI_OPERATION_NAME_FIELD]; - if (operationName) { - return String(operationName); - } - if (rootSpan.op) { - return rootSpan.op; - } - return UNKNOWN_OPERATION; -} - -function determineTraceName(rootSpan: Span): string { - const agentName = rootSpan.data?.[GEN_AI_AGENT_NAME_FIELD]; - if (agentName) { - return String(agentName); - } - return rootSpan.description || rootSpan.op || DEFAULT_TRACE_NAME; -} - -/** - * Sums token usage across every span in the tree so an agent root with multiple - * chat turns reports the total, not just the first turn's usage. - */ -function extractTokenUsage(spans: Span[]): { promptTokens?: number; completionTokens?: number } { - let promptTokens: number | undefined; - let completionTokens: number | undefined; - - for (const span of spans) { - if (!span.data) continue; - - if (span.data[GEN_AI_USAGE_INPUT_TOKENS_FIELD] !== undefined) { - promptTokens = (promptTokens ?? 0) + Number(span.data[GEN_AI_USAGE_INPUT_TOKENS_FIELD]); - } - - if (span.data[GEN_AI_USAGE_OUTPUT_TOKENS_FIELD] !== undefined) { - completionTokens = (completionTokens ?? 0) + Number(span.data[GEN_AI_USAGE_OUTPUT_TOKENS_FIELD]); - } - } - - return { promptTokens, completionTokens }; -} - -function formatTokensDisplay(promptTokens?: number, completionTokens?: number): string { - if (promptTokens !== undefined && completionTokens !== undefined) { - return `${promptTokens} / ${completionTokens}`; - } - if (promptTokens !== undefined) { - return `${promptTokens} / ?`; - } - if (completionTokens !== undefined) { - return `? / ${completionTokens}`; - } - return UNKNOWN_OPERATION; -} - -/** - * Normalizes message content from a string or an array of content blocks to a - * plain string. Only text blocks are surfaced; images/files would need UI work. - */ -function normalizeMessageContent(content: unknown): string { - if (typeof content === "string") return content; - if (Array.isArray(content)) { - return content - .filter( - (block): block is { type: string; text: string } => block?.type === "text" && typeof block.text === "string", - ) - .map(block => block.text) - .join(""); - } - return ""; -} - -function extractMetadata(span: Span, trace: SpotlightAITrace) { - if (!span.data) return; - - if (trace.metadata.modelId === undefined) { - const modelId = span.data[GEN_AI_REQUEST_MODEL_FIELD] ?? span.data[GEN_AI_RESPONSE_MODEL_FIELD]; - if (modelId !== undefined) { - trace.metadata.modelId = String(modelId); - } - } - - if (trace.metadata.modelProvider === undefined && span.data[GEN_AI_SYSTEM_FIELD] !== undefined) { - trace.metadata.modelProvider = String(span.data[GEN_AI_SYSTEM_FIELD]); - } -} - -function extractPromptData(span: Span, trace: SpotlightAITrace) { - if (!span.data || trace.prompt) return; - - const promptMessages = span.data[GEN_AI_REQUEST_MESSAGES_FIELD]; - if (promptMessages === undefined) return; - - const system = span.data[GEN_AI_SYSTEM_INSTRUCTIONS_FIELD]; - const systemText = system !== undefined ? String(system) : undefined; - - try { - const messages = typeof promptMessages === "string" ? JSON.parse(promptMessages) : promptMessages; - if (Array.isArray(messages)) { - const normalizedMessages = messages.map((msg: { role: string; content: unknown }) => ({ - ...msg, - content: normalizeMessageContent(msg.content), - })); - trace.prompt = { messages: normalizedMessages, system: systemText }; - return; - } - } catch { - // fall through to the raw representation below - } - - trace.prompt = { messages: [{ role: "unknown", content: String(promptMessages) }], system: systemText }; -} - -function extractFinishReason(finishReasons: unknown): string | undefined { - if (Array.isArray(finishReasons) && finishReasons.length > 0) { - return String(finishReasons[0]); - } - if (typeof finishReasons === "string" && finishReasons) { - return finishReasons; - } - return undefined; -} - -function extractResponseData(span: Span, trace: SpotlightAITrace, state: { finishReasonFromText: boolean }) { - if (!span.data) return; - - trace.response = trace.response || {}; - - // Spans arrive in tree pre-order, so for an agent root with several chat - // children the last span carrying a response wins — surfacing the latest - // model turn rather than the first. - const responseText = span.data[GEN_AI_RESPONSE_TEXT_FIELD]; - const text = Array.isArray(responseText) - ? responseText.map(String).join("") - : responseText - ? String(responseText) - : ""; - - const finishReason = extractFinishReason(span.data[GEN_AI_RESPONSE_FINISH_REASONS_FIELD]); - - if (text) { - trace.response.text = text; - // Re-pair the finish reason with the turn that produced the surfaced text, - // even overwriting one an earlier text-less turn contributed, so the text - // and its reason always come from the same turn. - trace.response.finishReason = finishReason; - state.finishReasonFromText = true; - } else if (finishReason !== undefined && !state.finishReasonFromText) { - // SDKs omit gen_ai.response.text when PII recording is off, and tool-call - // turns carry a reason with no text — still surface the reason, but never - // let it override one already paired with a text turn. - trace.response.finishReason = finishReason; - } - - const toolCalls = span.data[GEN_AI_RESPONSE_TOOL_CALLS_FIELD]; - if (toolCalls) { - try { - trace.response.toolCalls = typeof toolCalls === "string" ? JSON.parse(toolCalls) : (toolCalls as AIToolCall[]); - } catch { - // ignore unparseable tool call payloads - } - } -} - -function extractToolCallData(span: Span, trace: SpotlightAITrace) { - if (!span.data) return; - - const toolName = span.data[GEN_AI_TOOL_NAME_FIELD]; - if (!toolName) return; - - const toolCallId = span.data[GEN_AI_TOOL_CALL_ID_FIELD]; - const toolCall: Partial = { - toolCallId: toolCallId ? String(toolCallId) : span.span_id, - toolName: String(toolName), - args: {} as Record, - }; - - const toolInput = span.data[GEN_AI_TOOL_INPUT_FIELD]; - if (toolInput) { - try { - toolCall.args = typeof toolInput === "string" ? JSON.parse(toolInput) : (toolInput as Record); - } catch { - toolCall.args = { input: toolInput }; - } - } - - const toolOutput = span.data[GEN_AI_TOOL_OUTPUT_FIELD]; - if (toolOutput) { - try { - toolCall.result = typeof toolOutput === "string" ? JSON.parse(toolOutput) : toolOutput; - } catch { - toolCall.result = String(toolOutput); - } - } - - trace.toolCalls.push(toolCall as AIToolCall); -} +export * from "../../../../../../telemetry/ai/genAI.ts"; diff --git a/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/vercelAISDK.ts b/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/vercelAISDK.ts index d74543750..b79e1d349 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/vercelAISDK.ts +++ b/packages/spotlight/src/ui/telemetry/components/insights/aiTraces/sdks/vercelAISDK.ts @@ -1,467 +1 @@ -import { getAllSpansInTree } from "@spotlight/ui/telemetry/store/helpers"; -import type { AILibraryHandler, AIToolCall, Span, SpotlightAITrace } from "@spotlight/ui/telemetry/types"; - -// https://ai-sdk.dev/docs/ai-sdk-core/telemetry -const AI_SPAN_OP_PREFIX = "gen_ai."; -const AI_OPERATION_ID_FIELD = "vercel.ai.operationId"; -const AI_OPERATION_NAME_FIELD = "operation.name"; - -// Tool call related constants (OpenTelemetry semantic conventions) -const AI_TOOL_CALL_OPERATION = "ai.toolCall"; -const AI_EXECUTE_TOOL_OPERATION = "gen_ai.execute_tool"; -const GEN_AI_TOOL_NAME_FIELD = "gen_ai.tool.name"; -const GEN_AI_TOOL_CALL_ID_FIELD = "gen_ai.tool.call.id"; -const GEN_AI_TOOL_INPUT_FIELD = "gen_ai.tool.input"; -const GEN_AI_TOOL_OUTPUT_FIELD = "gen_ai.tool.output"; - -const AI_MODEL_ID_FIELD = "vercel.ai.model.id"; -const AI_MODEL_PROVIDER_FIELD = "vercel.ai.model.provider"; -const AI_SETTINGS_MAX_RETRIES_FIELD = "vercel.ai.settings.maxRetries"; -const AI_SETTINGS_MAX_STEPS_FIELD = "vercel.ai.settings.maxSteps"; -const AI_TELEMETRY_FUNCTION_ID_FIELD = "vercel.ai.telemetry.functionId"; -const AI_TELEMETRY_METADATA_PREFIX = "vercel.ai.telemetry.metadata."; -const AI_PROMPT_FIELD = "vercel.ai.prompt"; -const AI_PROMPT_MESSAGES_FIELD = "gen_ai.request.messages"; -const AI_RESPONSE_FINISH_REASON_FIELD = "vercel.ai.response.finishReason"; -const GEN_AI_RESPONSE_FINISH_REASONS_FIELD = "gen_ai.response.finish_reasons"; -const AI_RESPONSE_TEXT_FIELD = "gen_ai.response.text"; -const AI_RESPONSE_TOOL_CALLS_FIELD = "vercel.ai.response.toolCalls"; - -// v1 operation names -const AI_STREAM_TEXT_OPERATION = "ai.streamText"; -const AI_GENERATE_TEXT_OPERATION = "ai.generateText"; - -// v2 operation names (OpenTelemetry semantic conventions) -const AI_INVOKE_AGENT_OPERATION = "gen_ai.invoke_agent"; -const AI_STREAM_TEXT_V2_OPERATION = "gen_ai.stream_text"; - -// v2 field names -const GEN_AI_PROMPT_FIELD = "gen_ai.prompt"; -const GEN_AI_FUNCTION_ID_FIELD = "gen_ai.function_id"; -const GEN_AI_REQUEST_MODEL_FIELD = "gen_ai.request.model"; -const GEN_AI_RESPONSE_MODEL_FIELD = "gen_ai.response.model"; -const GEN_AI_SYSTEM_FIELD = "gen_ai.system"; -const GEN_AI_OPERATION_NAME_FIELD = "gen_ai.operation.name"; - -const AI_USAGE_PROMPT_TOKENS_FIELD = "vercel.ai.usage.promptTokens"; -const AI_USAGE_COMPLETION_TOKENS_FIELD = "vercel.ai.usage.completionTokens"; -const GEN_AI_USAGE_INPUT_TOKENS_FIELD = "gen_ai.usage.input_tokens"; -const GEN_AI_USAGE_OUTPUT_TOKENS_FIELD = "gen_ai.usage.output_tokens"; - -const TOKEN_FIELDS = { - PROMPT: [AI_USAGE_PROMPT_TOKENS_FIELD, GEN_AI_USAGE_INPUT_TOKENS_FIELD], - COMPLETION: [AI_USAGE_COMPLETION_TOKENS_FIELD, GEN_AI_USAGE_OUTPUT_TOKENS_FIELD], -} as const; - -// Other constants -const DEFAULT_TRACE_NAME = "AI Interaction"; -const UNKNOWN_OPERATION = "N/A"; - -// Operation to badge mapping -const OPERATION_BADGE_MAP = new Map([ - [AI_STREAM_TEXT_OPERATION, "Stream Text"], - [AI_GENERATE_TEXT_OPERATION, "Generate Text"], - [AI_INVOKE_AGENT_OPERATION, "Invoke Agent"], - [AI_STREAM_TEXT_V2_OPERATION, "Stream Text"], -]); - -// Field mappings: first element is v2, subsequent are v1 alternatives -const AI_FIELD_MAPPINGS = { - MODEL_ID: [AI_MODEL_ID_FIELD, GEN_AI_REQUEST_MODEL_FIELD, GEN_AI_RESPONSE_MODEL_FIELD], - MODEL_PROVIDER: [AI_MODEL_PROVIDER_FIELD, GEN_AI_SYSTEM_FIELD], - FUNCTION_ID: [AI_TELEMETRY_FUNCTION_ID_FIELD, GEN_AI_FUNCTION_ID_FIELD], - PROMPT_TOKENS: [AI_USAGE_PROMPT_TOKENS_FIELD, GEN_AI_USAGE_INPUT_TOKENS_FIELD], - COMPLETION_TOKENS: [AI_USAGE_COMPLETION_TOKENS_FIELD, GEN_AI_USAGE_OUTPUT_TOKENS_FIELD], - PROMPT: [GEN_AI_PROMPT_FIELD, AI_PROMPT_FIELD], - OPERATION_ID: [AI_OPERATION_ID_FIELD, AI_OPERATION_NAME_FIELD, GEN_AI_OPERATION_NAME_FIELD], -} as const; - -/** - * Extracts a field value from span data, checking v1 fields first then v2 alternatives. - */ -function extractFieldFromSpan( - data: Record, - fieldKey: keyof typeof AI_FIELD_MAPPINGS, -): unknown | undefined { - const fields = AI_FIELD_MAPPINGS[fieldKey]; - for (const field of fields) { - if (data[field] !== undefined) { - return data[field]; - } - } - return undefined; -} - -export const vercelAISDKHandler: AILibraryHandler = { - id: "vercel-ai-sdk", - name: "Vercel AI SDK", - - canHandleSpan: (span: Span): boolean => { - return !!span.op?.toLowerCase().startsWith(AI_SPAN_OP_PREFIX); - }, - - extractRootSpans: (spans: Span[]): Span[] => { - const resultRoots: Span[] = []; - - const findAndCollectAIRoots = (spansToSearch: Span[]) => { - for (const currentSpan of spansToSearch) { - if (currentSpan.op?.toLowerCase().startsWith(AI_SPAN_OP_PREFIX)) { - resultRoots.push(currentSpan); - } else if (currentSpan.children?.length) { - findAndCollectAIRoots(currentSpan.children); - } - } - }; - - findAndCollectAIRoots(spans); - return resultRoots; - }, - - processTrace: (rootSpan: Span): SpotlightAITrace => { - const allSpans = getAllSpansInTree(rootSpan); - const tokenUsage = extractTokenUsage(allSpans); - const operationInfo = determineOperation(allSpans, rootSpan); - const traceName = determineTraceName(rootSpan, operationInfo); - - const durationMs = rootSpan.timestamp - rootSpan.start_timestamp; - - // Initialize the trace with basic properties - const trace: SpotlightAITrace = { - id: rootSpan.span_id, - name: traceName, - operation: operationInfo.operation, - timestamp: rootSpan.start_timestamp, - durationMs, - tokensDisplay: formatTokensDisplay(tokenUsage.promptTokens, tokenUsage.completionTokens), - promptTokens: tokenUsage.promptTokens, - completionTokens: tokenUsage.completionTokens, - hasToolCall: operationInfo.hasToolCall, - rawSpan: rootSpan, - metadata: { - metadata: {}, - }, - toolCalls: [], - }; - - // Parse detailed span data - parseSpanData(allSpans, trace); - - return trace; - }, - - getDisplayTitle: (trace: SpotlightAITrace): string => { - return trace.name || trace.operation || `AI Trace ${trace.id.substring(0, 8)}`; - }, - - getTypeBadge: (trace: SpotlightAITrace): string => { - if (trace.hasToolCall) { - return "Tool-Call"; - } - - return OPERATION_BADGE_MAP.get(trace.operation) ?? trace.operation.replace(/^(ai\.|gen_ai\.)/, ""); - }, - - getTokensDisplay: (trace: SpotlightAITrace): string => { - return trace.tokensDisplay; - }, -}; - -function extractTokenUsage(spans: Span[]): { promptTokens?: number; completionTokens?: number } { - let promptTokens: number | undefined; - let completionTokens: number | undefined; - - for (const span of spans) { - if (!span.data) continue; - - if (promptTokens === undefined) { - for (const field of TOKEN_FIELDS.PROMPT) { - const value = span.data[field]; - if (value !== undefined) { - promptTokens = Number(value); - break; - } - } - } - - if (completionTokens === undefined) { - for (const field of TOKEN_FIELDS.COMPLETION) { - const value = span.data[field]; - if (value !== undefined) { - completionTokens = Number(value); - break; - } - } - } - - if (promptTokens !== undefined && completionTokens !== undefined) { - break; - } - } - - return { promptTokens, completionTokens }; -} - -function determineOperation( - spans: Span[], - rootSpan: Span, -): { operation: string; hasToolCall: boolean; toolCallName?: string } { - let operation = UNKNOWN_OPERATION; - let hasToolCall = false; - let toolCallName: string | undefined; - let foundToolCallAsOperationId = false; - - for (const span of spans) { - if (span.op === AI_TOOL_CALL_OPERATION || span.op === AI_EXECUTE_TOOL_OPERATION) { - hasToolCall = true; - } - - if (!span.data) continue; - - const operationId = extractFieldFromSpan(span.data, "OPERATION_ID") as string | undefined; - - // Handle tool call operation - if (operationId === AI_TOOL_CALL_OPERATION) { - operation = AI_TOOL_CALL_OPERATION; - foundToolCallAsOperationId = true; - hasToolCall = true; - - if (span.data[GEN_AI_TOOL_NAME_FIELD]) { - toolCallName = String(span.data[GEN_AI_TOOL_NAME_FIELD]); - } - } - - // Handle other operations (prioritize root and direct children) - if (!foundToolCallAsOperationId && operationId && operation === UNKNOWN_OPERATION) { - const isHighPriority = - span.span_id === rootSpan.span_id || rootSpan.children?.some(child => child.span_id === span.span_id); - - if (isHighPriority) { - operation = operationId; - } - } - } - - // Fallback logic - if (operation === UNKNOWN_OPERATION) { - if (rootSpan.op) { - operation = rootSpan.op; - } else if (rootSpan.description) { - operation = rootSpan.description; - } - } - - return { operation, hasToolCall, toolCallName }; -} - -function determineTraceName(rootSpan: Span, operationInfo: { toolCallName?: string }): string { - if (operationInfo.toolCallName) { - return operationInfo.toolCallName; - } - - return rootSpan.description || rootSpan.op || DEFAULT_TRACE_NAME; -} - -function formatTokensDisplay(promptTokens?: number, completionTokens?: number): string { - if (promptTokens !== undefined && completionTokens !== undefined) { - return `${promptTokens} / ${completionTokens}`; - } - if (promptTokens !== undefined) { - return `${promptTokens} / ?`; - } - if (completionTokens !== undefined) { - return `? / ${completionTokens}`; - } - return UNKNOWN_OPERATION; -} - -/** - * Normalizes message content from v1 (string) or v2 (array of content blocks) format to a string. - * v1: content = "text string" - * v2: content = [{ type: "text", text: "text string" }] - * - * Only extracts text content blocks. Other block types (image, file, tool-call) - * would require UI changes to display properly and are filtered out for now. - */ -function normalizeMessageContent(content: unknown): string { - if (typeof content === "string") return content; - if (Array.isArray(content)) { - // Only extract text content - images/files would need UI changes to display - return content - .filter( - (block): block is { type: string; text: string } => block.type === "text" && typeof block.text === "string", - ) - .map(block => block.text) - .join(""); - } - return ""; -} - -function parseSpanData(spans: Span[], trace: SpotlightAITrace) { - for (const span of spans) { - if (!span.data) continue; - - extractAIMetadata(span, trace); - extractTelemetryMetadata(span, trace); - extractPromptData(span, trace); - extractResponseData(span, trace); - extractToolCallData(span, trace); - } -} - -function extractAIMetadata(span: Span, trace: SpotlightAITrace) { - if (!span.data) return; - - // Model ID - if (trace.metadata.modelId === undefined) { - const modelId = extractFieldFromSpan(span.data, "MODEL_ID"); - if (modelId !== undefined) { - trace.metadata.modelId = String(modelId); - } - } - - // Model Provider - if (trace.metadata.modelProvider === undefined) { - const provider = extractFieldFromSpan(span.data, "MODEL_PROVIDER"); - if (provider !== undefined) { - trace.metadata.modelProvider = String(provider); - } - } - - // Function ID - if (trace.metadata.functionId === undefined) { - const functionId = extractFieldFromSpan(span.data, "FUNCTION_ID"); - if (functionId !== undefined) { - trace.metadata.functionId = String(functionId); - } - } - - if (span.data[AI_SETTINGS_MAX_RETRIES_FIELD]) { - trace.metadata.maxRetries = Number(span.data[AI_SETTINGS_MAX_RETRIES_FIELD]); - } - - if (span.data[AI_SETTINGS_MAX_STEPS_FIELD]) { - trace.metadata.maxSteps = Number(span.data[AI_SETTINGS_MAX_STEPS_FIELD]); - } - - // Token usage - use === undefined to allow 0 values - if (trace.metadata.promptTokens === undefined) { - const promptTokens = extractFieldFromSpan(span.data, "PROMPT_TOKENS"); - if (promptTokens !== undefined) { - trace.metadata.promptTokens = Number(promptTokens); - } - } - - if (trace.metadata.completionTokens === undefined) { - const completionTokens = extractFieldFromSpan(span.data, "COMPLETION_TOKENS"); - if (completionTokens !== undefined) { - trace.metadata.completionTokens = Number(completionTokens); - } - } -} - -function extractTelemetryMetadata(span: Span, trace: SpotlightAITrace) { - if (!span.data) return; - - for (const [key, value] of Object.entries(span.data)) { - if (key.startsWith(AI_TELEMETRY_METADATA_PREFIX)) { - const metadataKey = key.replace(AI_TELEMETRY_METADATA_PREFIX, ""); - trace.metadata.metadata[metadataKey] = value; - } - } -} - -function extractPromptData(span: Span, trace: SpotlightAITrace) { - if (!span.data) return; - - const promptField = extractFieldFromSpan(span.data, "PROMPT"); - if (promptField && !trace.prompt) { - try { - const parsed = JSON.parse(String(promptField)); - // Normalize message content from v2 array format to string - if (parsed.messages && Array.isArray(parsed.messages)) { - parsed.messages = parsed.messages.map((msg: { role: string; content: unknown }) => ({ - ...msg, - content: normalizeMessageContent(msg.content), - })); - } - trace.prompt = parsed; - } catch { - trace.prompt = { messages: [{ role: "unknown", content: String(promptField) }] }; - } - } - - const promptMessages = span.data[AI_PROMPT_MESSAGES_FIELD]; - if (promptMessages && !trace.prompt) { - try { - const messages = JSON.parse(String(promptMessages)); - // Normalize message content from v2 array format to string - const normalizedMessages = messages.map((msg: { role: string; content: unknown }) => ({ - ...msg, - content: normalizeMessageContent(msg.content), - })); - trace.prompt = { messages: normalizedMessages }; - } catch { - trace.prompt = { messages: [{ role: "unknown", content: String(promptMessages) }] }; - } - } -} - -function extractResponseData(span: Span, trace: SpotlightAITrace) { - if (!span.data) return; - - trace.response = trace.response || {}; - - const finishReason = span.data[AI_RESPONSE_FINISH_REASON_FIELD]; - if (finishReason) { - trace.response.finishReason = String(finishReason); - } - - const finishReasons = span.data[GEN_AI_RESPONSE_FINISH_REASONS_FIELD]; - if (!trace.response.finishReason && Array.isArray(finishReasons) && finishReasons.length > 0) { - trace.response.finishReason = String(finishReasons[0]); - } - - const responseText = span.data[AI_RESPONSE_TEXT_FIELD]; - if (responseText) { - trace.response.text = String(responseText); - } - - const toolCalls = span.data[AI_RESPONSE_TOOL_CALLS_FIELD]; - if (toolCalls) { - trace.response.toolCalls = JSON.parse(String(toolCalls)); - } -} - -function extractToolCallData(span: Span, trace: SpotlightAITrace) { - if (!span.data) return; - - const toolName = span.data[GEN_AI_TOOL_NAME_FIELD]; - if (!toolName) return; - - const toolCallId = span.data[GEN_AI_TOOL_CALL_ID_FIELD]; - const toolCall: Partial = { - toolCallId: toolCallId ? String(toolCallId) : span.span_id, - toolName: String(toolName), - args: {} as Record, - }; - - const toolInput = span.data[GEN_AI_TOOL_INPUT_FIELD]; - if (toolInput) { - try { - toolCall.args = typeof toolInput === "string" ? JSON.parse(toolInput) : (toolInput as Record); - } catch { - toolCall.args = { input: toolInput }; - } - } - - const toolOutput = span.data[GEN_AI_TOOL_OUTPUT_FIELD]; - if (toolOutput) { - try { - toolCall.result = typeof toolOutput === "string" ? JSON.parse(toolOutput) : toolOutput; - } catch { - toolCall.result = String(toolOutput); - } - } - - trace.toolCalls.push(toolCall as AIToolCall); -} +export * from "../../../../../../telemetry/ai/vercelAISDK.ts"; diff --git a/packages/spotlight/src/ui/telemetry/components/insights/webVitals/WebVitalsDetail.tsx b/packages/spotlight/src/ui/telemetry/components/insights/webVitals/WebVitalsDetail.tsx index 759c2ca63..1a6e5abc5 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/webVitals/WebVitalsDetail.tsx +++ b/packages/spotlight/src/ui/telemetry/components/insights/webVitals/WebVitalsDetail.tsx @@ -1,14 +1,9 @@ -import { PERFORMANCE_SCORE_PROFILES } from "@spotlight/ui/telemetry/constants"; import { useSentryEvents } from "@spotlight/ui/telemetry/data/useSentryEvents"; -import type { - MetricScoreProps, - MetricWeightsProps, - SentryEventWithPerformanceData, -} from "@spotlight/ui/telemetry/types"; +import type { MetricScoreProps, MetricWeightsProps } from "@spotlight/ui/telemetry/types"; import { getFormattedDuration } from "@spotlight/ui/telemetry/utils/duration"; import Breadcrumbs from "@spotlight/ui/ui/breadcrumbs"; import { useParams } from "react-router-dom"; -import { normalizePerformanceScore } from "../../../utils/webVitals"; +import { getWebVitals } from "../../../../../telemetry/selectors/webVitals.ts"; import EmptyState from "../../shared/EmptyState"; import PerformanceChart from "./PerformanceChart"; @@ -16,17 +11,7 @@ const WebVitalsDetail = () => { const events = useSentryEvents(); const { page } = useParams(); - let measurementEvent: SentryEventWithPerformanceData | undefined; - for (const event of events) { - if (event.event_id === page && event.measurements && event?.contexts?.trace?.op === "pageload") { - const updatedEvent = { ...event }; - normalizePerformanceScore(updatedEvent, PERFORMANCE_SCORE_PROFILES); - if (updatedEvent.measurements["score.total"] != null) { - measurementEvent = updatedEvent as unknown as SentryEventWithPerformanceData; - break; - } - } - } + const measurementEvent = getWebVitals(events).find(event => event.event_id === page); if (page && measurementEvent) { const metricScore: MetricScoreProps = { diff --git a/packages/spotlight/src/ui/telemetry/components/insights/webVitals/index.tsx b/packages/spotlight/src/ui/telemetry/components/insights/webVitals/index.tsx index 0f7a9ae1d..84c6a6d98 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/webVitals/index.tsx +++ b/packages/spotlight/src/ui/telemetry/components/insights/webVitals/index.tsx @@ -1,11 +1,7 @@ import { ReactComponent as Sort } from "@spotlight/ui/assets/sort.svg"; import { ReactComponent as SortDown } from "@spotlight/ui/assets/sortDown.svg"; import { cn } from "@spotlight/ui/lib/cn"; -import { - PERFORMANCE_SCORE_PROFILES, - WEB_VITALS_HEADERS, - WEB_VITALS_SORT_KEYS, -} from "@spotlight/ui/telemetry/constants"; +import { WEB_VITALS_HEADERS, WEB_VITALS_SORT_KEYS } from "@spotlight/ui/telemetry/constants"; import { useSentryEvents } from "@spotlight/ui/telemetry/data/useSentryEvents"; import useSort from "@spotlight/ui/telemetry/hooks/useSort"; import type { SentryEventWithPerformanceData } from "@spotlight/ui/telemetry/types"; @@ -13,7 +9,7 @@ import { getFormattedDuration } from "@spotlight/ui/telemetry/utils/duration"; import Table from "@spotlight/ui/ui/table"; import { type KeyboardEvent, useMemo } from "react"; import { useNavigate } from "react-router-dom"; -import { normalizePerformanceScore } from "../../../utils/webVitals"; +import { getWebVitals } from "../../../../../telemetry/selectors/webVitals.ts"; import EmptyState from "../../shared/EmptyState"; type SentryEventComparator = (a: SentryEventWithPerformanceData, b: SentryEventWithPerformanceData) => number; @@ -40,25 +36,7 @@ const WebVitals = () => { const measurementEvents: SentryEventWithPerformanceData[] = useMemo(() => { const compareEvents = COMPARATORS[sort.active] || COMPARATORS[WEB_VITALS_SORT_KEYS.score]; - return ( - events - // TODO: Skipping this check because of not getting all required metrics - // && !PERFORMANCE_SCORE_PROFILES.profiles[0].scoreComponents.some(c => { - // return ( - // !Object.prototype.hasOwnProperty.call(event.measurements, c.measurement) && - // Math.abs(c.weight) >= Number.EPSILON && - // !c.optional - // ); - // }) - .filter(event => event.measurements && event?.contexts?.trace?.op === "pageload") - .map(event => { - const updatedEvent = { ...event }; - normalizePerformanceScore(updatedEvent, PERFORMANCE_SCORE_PROFILES); - return updatedEvent as unknown as SentryEventWithPerformanceData; - }) - .filter(event => event.measurements["score.total"] != null) - .sort((a, b) => (sort.asc ? compareEvents(a, b) : compareEvents(b, a))) - ); + return getWebVitals(events).sort((a, b) => (sort.asc ? compareEvents(a, b) : compareEvents(b, a))); }, [events, sort]); const handleRowClick = (measurementEvent: SentryEventWithPerformanceData) => { diff --git a/packages/spotlight/src/ui/telemetry/constants.ts b/packages/spotlight/src/ui/telemetry/constants.ts index cc91afb08..0113ba8e1 100644 --- a/packages/spotlight/src/ui/telemetry/constants.ts +++ b/packages/spotlight/src/ui/telemetry/constants.ts @@ -3,7 +3,7 @@ import type { NanovisTreeNode } from "./types"; /** Base path the telemetry UI is mounted under (see `App.tsx` `/telemetry/*`). */ export const TELEMETRY_BASE_URL = "/telemetry"; -export const DB_SPAN_REGEX = /^db(?:\.[A-Za-z]+)*$/; +export { DB_SPAN_REGEX } from "../../telemetry/selectors/database.ts"; export const AGGREGATE_CALL_PROFILES_SORT_KEYS = { functionName: "FUNCTION_NAME", @@ -260,66 +260,7 @@ export const WEB_VITALS_HEADERS = [ export type WebVitals = "lcp" | "fcp" | "cls" | "ttfb" | "fid"; -export const PERFORMANCE_SCORE_PROFILES = { - profiles: [ - { - name: "Chrome", - scoreComponents: [ - { measurement: "fcp", weight: 0.15, p10: 900.0, p50: 1600.0, optional: false }, - { measurement: "lcp", weight: 0.3, p10: 1200.0, p50: 2400.0, optional: false }, - { measurement: "fid", weight: 0.3, p10: 100.0, p50: 300.0, optional: true }, - { measurement: "cls", weight: 0.15, p10: 0.1, p50: 0.25, optional: false }, - { measurement: "ttfb", weight: 0.1, p10: 200.0, p50: 400.0, optional: false }, - ], - condition: { op: "eq", name: "event.contexts.browser.name", value: "Chrome" }, - }, - // TODO: Currently not getting browser data in events, so made these comments and sticked to chrome always. - // { - // "name": "Firefox", - // "scoreComponents": [ - // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, - // {"measurement": "lcp", "weight": 0.0, "p10": 1200.0, "p50": 2400.0, "optional": false}, - // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, - // {"measurement": "cls", "weight": 0.0, "p10": 0.1, "p50": 0.25, "optional": false}, - // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} - // ], - // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Firefox"} - // }, - // { - // "name": "Safari", - // "scoreComponents": [ - // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, - // {"measurement": "lcp", "weight": 0.0, "p10": 1200.0, "p50": 2400.0, "optional": false}, - // {"measurement": "fid", "weight": 0.0, "p10": 100.0, "p50": 300.0, "optional": true}, - // {"measurement": "cls", "weight": 0.0, "p10": 0.1, "p50": 0.25, "optional": false}, - // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} - // ], - // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Safari"} - // }, - // { - // "name": "Edge", - // "scoreComponents": [ - // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, - // {"measurement": "lcp", "weight": 0.30, "p10": 1200.0, "p50": 2400.0, "optional": false}, - // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, - // {"measurement": "cls", "weight": 0.15, "p10": 0.1, "p50": 0.25, "optional": false}, - // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} - // ], - // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Edge"} - // }, - // { - // "name": "Opera", - // "scoreComponents": [ - // {"measurement": "fcp", "weight": 0.15, "p10": 900.0, "p50": 1600.0, "optional": false}, - // {"measurement": "lcp", "weight": 0.30, "p10": 1200.0, "p50": 2400.0, "optional": false}, - // {"measurement": "fid", "weight": 0.30, "p10": 100.0, "p50": 300.0, "optional": true}, - // {"measurement": "cls", "weight": 0.15, "p10": 0.1, "p50": 0.25, "optional": false}, - // {"measurement": "ttfb", "weight": 0.10, "p10": 200.0, "p50": 400.0, "optional": false} - // ], - // "condition": {"op": "eq", "name": "event.contexts.browser.name", "value": "Opera"} - // } - ], -}; +export { PERFORMANCE_SCORE_PROFILES } from "../../telemetry/constants/performance.ts"; export const LOG_LEVEL_COLORS: Record = { trace: "text-gray-500", diff --git a/packages/spotlight/src/ui/telemetry/constants/sentry.ts b/packages/spotlight/src/ui/telemetry/constants/sentry.ts index c7ee6224c..0c9b14b56 100644 --- a/packages/spotlight/src/ui/telemetry/constants/sentry.ts +++ b/packages/spotlight/src/ui/telemetry/constants/sentry.ts @@ -1,13 +1 @@ -export const ERROR_EVENT_TYPES = new Set(["event", "error"]); -export const TRACE_EVENT_TYPES = new Set(["transaction"]); -export const PROFILE_EVENT_TYPES = new Set(["profile", "profile_chunk"]); -export const LOG_EVENT_TYPES = new Set(["log"]); -export const FEEDBACK_EVENT_TYPES = new Set(["feedback"]); -export const ATTACHMENT_EVENT_TYPES = new Set(["attachment"]); -export const SUPPORTED_EVENT_TYPES = new Set([ - ...ERROR_EVENT_TYPES, - ...TRACE_EVENT_TYPES, - ...PROFILE_EVENT_TYPES, - ...LOG_EVENT_TYPES, - ...FEEDBACK_EVENT_TYPES, -]); +export * from "../../../telemetry/constants/sentry.ts"; diff --git a/packages/spotlight/src/ui/telemetry/data/profiles.ts b/packages/spotlight/src/ui/telemetry/data/profiles.ts index acab376e4..8475b23a5 100644 --- a/packages/spotlight/src/ui/telemetry/data/profiles.ts +++ b/packages/spotlight/src/ui/telemetry/data/profiles.ts @@ -1,206 +1 @@ -import { log } from "@spotlight/ui/lib/logger"; -import { generateUuidv4 } from "@spotlight/ui/lib/uuid"; -import type { SentryProfileWithTraceMeta } from "../store/types"; -import type { Span, Trace } from "../types"; -import { getFunctionNameFromFrame } from "../utils/profileUtils"; -import { compareSpans } from "../utils/traces"; - -/** - * Groups consequent spans with the same description and op into a single span per each level. - * Essentially a BFS traversal of the spans tree. - * @param spans Span[] A list of spans to consolidate, sorted by their start_timestamp - * @returns Span[] A list of spans with the same description and op consolidated into a single span - */ -function consolidateSpans(trace: Trace, spans: Span[]): Span[] { - const consolidatedSpans: Span[] = []; - let lastSpan = spans[0]; - let spanIdx = 1; - while (spanIdx < spans.length + 1) { - const span = spans[spanIdx]; - spanIdx += 1; - if (span && span.description === lastSpan.description && span.op === lastSpan.op) { - // Require the spans are sorted by start_timestamp - lastSpan.timestamp = span.timestamp; - if (span.children) { - if (lastSpan.children) { - for (const child of span.children) { - lastSpan.children.push(child); - } - lastSpan.children.sort(compareSpans); - } else { - lastSpan.children = span.children; - } - } - } else { - // Discard very short spans - if (lastSpan.timestamp - lastSpan.start_timestamp > 0) { - if (lastSpan.children) { - lastSpan.children = consolidateSpans(trace, lastSpan.children); - } - consolidatedSpans.push(lastSpan); - trace.spans.set(lastSpan.span_id, lastSpan); - } - lastSpan = span; - } - } - - return consolidatedSpans; -} - -// This is done per platform right now but we may want to make it use -// SDK or something more specific, especially for JS as `javascript` may -// mean browser, node, etc. -const SENTRY_FRAME_FILTER_PER_PLATFORM: Record< - string, - (this: SentryProfileWithTraceMeta["frames"], frameIdx: number) => boolean | undefined -> = { - python: function (frameIdx) { - return this[frameIdx].module?.startsWith("sentry_sdk."); - }, - javascript: function (frameIdx) { - const frame = this[frameIdx]; - const module = frame.module; - if (module) { - return module.startsWith("@sentry") || module.startsWith("@opentelemetry.instrumentation"); - } - // This one below is to match things like `http://localhost:3000/node_modules/.vite/deps/@sentry_react.js?v=6942e78f` etc. - return frame.abs_path ? /\/node_modules\/.*\/@(sentry|opentelemetry)[^a-z0-9]/.test(frame.abs_path) : false; - }, -}; - -export function getSpansFromProfile( - trace: Trace, - profile: SentryProfileWithTraceMeta, - parent_span_id: string | undefined, - startTs: number, - endTs: number, - threadIds: Set, -): Span[] { - threadIds.add(profile.active_thread_id); - - const sentryFrameFilter = profile.platform && SENTRY_FRAME_FILTER_PER_PLATFORM[profile.platform]; - // Try to fill in the gaps from profile data - const fillerSpans: Span[] = []; - for (let sampleIdx = 0; sampleIdx < profile.samples.length; sampleIdx++) { - const sample = profile.samples[sampleIdx]; - if (sample.thread_id && !threadIds.has(sample.thread_id)) { - continue; - } - const sampleTs = sample.start_timestamp; - if (sampleTs < startTs || sampleTs > endTs) { - continue; - } - const nextSample = profile.samples[sampleIdx + 1]; - const timestamp = nextSample ? nextSample.start_timestamp : endTs; - - if (timestamp > endTs) { - continue; - } - const commonAttributes = { - start_timestamp: sampleTs, - timestamp, - trace_id: trace.trace_id, - status: "ok", - tags: { source: "profile" }, - data: { - "thread.id": sample.thread_id, - "thread.name": profile.thread_metadata?.[sample.thread_id as keyof typeof profile.thread_metadata]?.name, - }, - }; - const sampleSpan: Span = { - span_id: generateUuidv4(), - parent_span_id, - ...commonAttributes, - op: "Thread", - description: - profile.thread_metadata?.[sample.thread_id as keyof typeof profile.thread_metadata]?.name || - `Thread ${sample.thread_id}`, - data: { - thread_id: sample.thread_id, - }, - }; - let currentSpan = sampleSpan; - const currentStack = profile.stacks[sample.stack_id]; - const lastSentryFrameIdx = sentryFrameFilter ? currentStack.findLastIndex(sentryFrameFilter, profile.frames) : 0; - for (let frameIdxIdx = lastSentryFrameIdx + 1; frameIdxIdx < currentStack.length; frameIdxIdx++) { - const frame = profile.frames[currentStack[frameIdxIdx]]; - // XXX: We may wanna skip frames that doesn't have `in_app` set to true - // that said it's better to have this as a dynamic filter - const [description, op] = getFunctionNameFromFrame(frame).split("@"); - const spanFromFrame: Span = { - span_id: generateUuidv4(), - parent_span_id: currentSpan.span_id, - ...commonAttributes, - op, - description, - data: { - ...frame, - }, - }; - currentSpan.children = [spanFromFrame]; - currentSpan = spanFromFrame; - } - fillerSpans.push(sampleSpan); - } - - if (!fillerSpans.length) { - return []; - } - const consolidated = consolidateSpans(trace, fillerSpans); - // Remove the extra layer of nesting if there is only one span which should be the "Thread" span - return (consolidated.length === 1 ? consolidated[0].children || [] : consolidated).filter( - span => span.timestamp - span.start_timestamp > 0 && span.timestamp <= endTs, - ); -} - -/** - * Modifies the spanTree in place recursively by adding spans from the - * profile data where there are gaps in the trace data. - * @param spanTree Span[] The tree of spans to graft profile spans into - */ -export function graftProfileSpans( - trace: Trace, - profile: SentryProfileWithTraceMeta, - spanTree: Span[] = trace.spanTree, - parent: Span | Trace = trace, -) { - log(`Grafting profile spans into trace ${trace.trace_id}`); - if (trace.profileGrafted) { - log(`Trace already has profile grafted ${trace.trace_id}`); - return; - } - - let idx = -1; - while (idx < spanTree.length) { - const span = spanTree[idx] as Span | undefined; - if (span?.tags?.source === "profile") { - idx += 1; - continue; - } - const nextSpan = spanTree[idx + 1]; - if (nextSpan?.tags?.source === "profile") { - idx += 1; - continue; - } - - const startTs = span ? span.timestamp : parent.start_timestamp; - const endTs = nextSpan ? nextSpan.start_timestamp : parent.timestamp; - const threadIds = new Set([span?.data?.threadId, nextSpan?.data?.threadId, parent?.data?.threadId]); - threadIds.delete(undefined); - if (endTs - startTs > 0) { - const fillers = getSpansFromProfile(trace, profile, parent.span_id, startTs, endTs, threadIds as Set); - if (fillers.length) { - spanTree.splice(idx + 1, 0, ...fillers); - idx += fillers.length; - } - } - if (span) { - span.children ??= []; - graftProfileSpans(trace, profile, span.children, span); - } - idx += 1; - } - // Only mark as grafted at the top level to avoid early quitting during - // recursive calls above for child spans - trace.profileGrafted = trace.spanTree === spanTree; -} +export * from "../../../telemetry/data/profiles.ts"; diff --git a/packages/spotlight/src/ui/telemetry/data/sentryEventsContext.test.tsx b/packages/spotlight/src/ui/telemetry/data/sentryEventsContext.test.tsx new file mode 100644 index 000000000..5fa56be55 --- /dev/null +++ b/packages/spotlight/src/ui/telemetry/data/sentryEventsContext.test.tsx @@ -0,0 +1,53 @@ +import { act, cleanup, render, screen } from "@testing-library/react"; +import { useContext } from "react"; +import { afterEach, expect, test } from "vitest"; +import useSentryStore from "../store"; +import { SentryEventsContext, SentryEventsContextProvider } from "./sentryEventsContext"; + +function Probe() { + const { events } = useContext(SentryEventsContext); + return {events.length}; +} +afterEach(() => { + cleanup(); + useSentryStore.getState().resetData(); +}); + +test("reset removes captured events from the React context", async () => { + useSentryStore.getState().resetData(); + render( + + + , + ); + await act(async () => { + await useSentryStore.getState().ingestEnvelope([{}, [[{ type: "event" }, { event_id: "captured", timestamp: 1 }]]]); + }); + expect(screen.getByTestId("event-count").textContent).toBe("1"); + act(() => useSentryStore.getState().resetData()); + expect(screen.getByTestId("event-count").textContent).toBe("0"); +}); + +test("raw-only envelope eviction releases old events held by the React context", async () => { + useSentryStore.getState().resetData(); + await useSentryStore + .getState() + .ingestEnvelope([ + { __spotlight_envelope_id: "old" }, + [[{ type: "event" }, { event_id: "captured", timestamp: 1 }]], + ]); + render( + + + , + ); + expect(screen.getByTestId("event-count").textContent).toBe("1"); + await act(async () => { + for (let index = 0; index < 500; index++) + await useSentryStore + .getState() + .ingestEnvelope([{ __spotlight_envelope_id: `raw-${index}` }, [[{ type: "unknown" }, {}]]]); + }); + expect(useSentryStore.getState().getEvents()).toEqual([]); + expect(screen.getByTestId("event-count").textContent).toBe("0"); +}); diff --git a/packages/spotlight/src/ui/telemetry/data/sentryEventsContext.tsx b/packages/spotlight/src/ui/telemetry/data/sentryEventsContext.tsx index 0a0bbe9ae..7a4109520 100644 --- a/packages/spotlight/src/ui/telemetry/data/sentryEventsContext.tsx +++ b/packages/spotlight/src/ui/telemetry/data/sentryEventsContext.tsx @@ -1,56 +1,12 @@ -import type { ReactNode } from "react"; -import React, { useEffect, useReducer } from "react"; +import { type ReactNode, createContext, useMemo } from "react"; import useSentryStore from "../store"; import type { SentryEvent } from "../types"; -interface SetEventsAction { - e: SentryEvent | SentryEvent[]; - action: string; -} - -interface SentryEventsContextProps { - events: SentryEvent[]; - setEvents: React.Dispatch; -} - -export const SentryEventsContext = React.createContext({ - events: [], - setEvents: () => {}, -}); - -function eventReducer(state: SentryEvent[], message: SetEventsAction): SentryEvent[] { - if (Array.isArray(message.e)) { - if (message.action === "RESET") { - return message.e; - } - } else { - if (message.action === "APPEND") { - return [message.e, ...state]; - } - } +export const SentryEventsContext = createContext<{ events: SentryEvent[] }>({ events: [] }); - return state; +/** Context mirrors committed SDK state so reset/retention release old event references. */ +export function SentryEventsContextProvider({ children }: { children: ReactNode }) { + const state = useSentryStore(); + const value = useMemo(() => ({ events: Array.from(state.eventsById.values()) }), [state]); + return {children}; } - -export const SentryEventsContextProvider: React.FC<{ - children: ReactNode; -}> = ({ children }) => { - const getEvents = useSentryStore(state => state.getEvents); - const subscribe = useSentryStore(state => state.subscribe); - const [events, setEvents] = useReducer(eventReducer, getEvents()); - - useEffect( - () => - subscribe("event", (e: SentryEvent) => { - setEvents({ action: "APPEND", e }); - }) as () => undefined, - [subscribe], - ); - - const contextValue: SentryEventsContextProps = { - events, - setEvents, - }; - - return {children}; -}; diff --git a/packages/spotlight/src/ui/telemetry/data/useSentrySpans.ts b/packages/spotlight/src/ui/telemetry/data/useSentrySpans.ts index cb22e11fb..b5a82de02 100644 --- a/packages/spotlight/src/ui/telemetry/data/useSentrySpans.ts +++ b/packages/spotlight/src/ui/telemetry/data/useSentrySpans.ts @@ -1,6 +1,6 @@ import { useContext } from "react"; +import { getSpans } from "../../../telemetry/selectors/spans.ts"; import useSentryStore from "../store"; -import type { Span, Trace } from "../types"; import { SentryEventsContext } from "./sentryEventsContext"; export function useSentryTraces() { @@ -9,27 +9,15 @@ export function useSentryTraces() { return getTraces().sort((a, b) => b.start_timestamp - a.start_timestamp); } -function spanReducer(acc: Span[], trace: Trace) { - for (const span of trace.spans.values()) { - acc.push(span); - } - return acc; -} - -function spanCountReducer(sum: number, trace: Trace) { - return sum + trace.spans.size; -} - export const useSentrySpans = () => { const allTraces = useSentryTraces(); - const allSpans: Span[] = allTraces.reduce(spanReducer, []); - return allSpans; + return getSpans(allTraces); }; export const useSentrySpanCounts = () => { - const allTraces = useSentryTraces(); + const spans = useSentrySpans(); return { - allSpans: allTraces.reduce(spanCountReducer, 0), + allSpans: spans.length, }; }; diff --git a/packages/spotlight/src/ui/telemetry/data/useSpotlightAITraces.ts b/packages/spotlight/src/ui/telemetry/data/useSpotlightAITraces.ts index 5436cb301..76dc5c236 100644 --- a/packages/spotlight/src/ui/telemetry/data/useSpotlightAITraces.ts +++ b/packages/spotlight/src/ui/telemetry/data/useSpotlightAITraces.ts @@ -1,70 +1,13 @@ import { useMemo } from "react"; -import { aiLibraries, extractAllAIRootSpans } from "../components/insights/aiTraces/sdks/aiLibraries"; +import { getAIRootSpans, getAITraces } from "../../../telemetry/selectors/ai.ts"; import type { Span, SpotlightAITrace } from "../types"; import { useSentryTraces } from "./useSentrySpans"; export function useAITraces(): Span[] { - const allTraces = useSentryTraces(); - - return useMemo(() => { - const rootSpans: Span[] = []; - - for (const trace of allTraces) { - if (!trace.spanTree) continue; - - for (const { span } of extractAllAIRootSpans(trace.spanTree)) { - rootSpans.push(span); - } - } - - return rootSpans; - }, [allTraces]); -} - -function processSpanAsTrace(span: Span): SpotlightAITrace { - if (!span?.span_id) { - return { - id: "error-invalid-span", - name: "Invalid Span", - operation: "error", - timestamp: 0, - durationMs: 0, - tokensDisplay: "N/A", - hasToolCall: false, - rawSpan: {} as Span, - metadata: { - metadata: {}, - }, - toolCalls: [], - }; - } - - for (const framework of aiLibraries) { - if (framework.canHandleSpan(span)) { - return framework.processTrace(span); - } - } - - return { - id: span.span_id, - name: span.description || span.op || "Unknown AI Interaction", - operation: "unknown", - timestamp: span.start_timestamp || 0, - durationMs: (span.timestamp || 0) - (span.start_timestamp || 0), - tokensDisplay: "N/A", - hasToolCall: false, - rawSpan: span, - metadata: { - metadata: {}, - }, - toolCalls: [], - }; + const traces = useSentryTraces(); + return useMemo(() => getAIRootSpans(traces), [traces]); } - export function useSpotlightAITraces(): SpotlightAITrace[] { - const aiRootSpans = useAITraces(); - - return useMemo(() => { - return aiRootSpans.map(processSpanAsTrace); - }, [aiRootSpans]); + const traces = useSentryTraces(); + return useMemo(() => getAITraces(traces), [traces]); } diff --git a/packages/spotlight/src/ui/telemetry/hooks/useErrorFiltering.test.tsx b/packages/spotlight/src/ui/telemetry/hooks/useErrorFiltering.test.tsx index 4fb0a6d5d..3b2c8c26c 100644 --- a/packages/spotlight/src/ui/telemetry/hooks/useErrorFiltering.test.tsx +++ b/packages/spotlight/src/ui/telemetry/hooks/useErrorFiltering.test.tsx @@ -3,7 +3,9 @@ import { describe, expect, it } from "vitest"; import type { SentryErrorEvent } from "../types"; import useErrorFiltering from "./useErrorFiltering"; -function makeError(overrides: Partial & { type: string; value: string }): SentryErrorEvent { +function makeError( + overrides: Partial> & { type: string; value: string }, +): SentryErrorEvent { const { type, value, ...rest } = overrides; return { event_id: `${type}-${value}`, diff --git a/packages/spotlight/src/ui/telemetry/index.tsx b/packages/spotlight/src/ui/telemetry/index.tsx index 3f65e9b9d..23117b223 100644 --- a/packages/spotlight/src/ui/telemetry/index.tsx +++ b/packages/spotlight/src/ui/telemetry/index.tsx @@ -18,7 +18,7 @@ export function Telemetry({ sidecarUrl }: TelemetryRouteProps) { const listener = useCallback((event: string): void => { log(`Received new ${SENTRY_CONTENT_TYPE} event`); const envelope = typeof event === "string" ? JSON.parse(event) : event; - useSentryStore.getState().pushEnvelope(envelope); + void useSentryStore.getState().ingestEnvelope(envelope); }, []); const contentTypeListeners = useMemo(() => { diff --git a/packages/spotlight/src/ui/telemetry/store/contextLines.test.ts b/packages/spotlight/src/ui/telemetry/store/contextLines.test.ts new file mode 100644 index 000000000..627ed5d4e --- /dev/null +++ b/packages/spotlight/src/ui/telemetry/store/contextLines.test.ts @@ -0,0 +1,103 @@ +import { afterEach, expect, test, vi } from "vitest"; + +const { fetchContext } = vi.hoisted(() => ({ fetchContext: vi.fn() })); +vi.mock("../utils/fetch", () => ({ getNativeFetchImplementation: () => fetchContext })); + +import useSentryStore from "./store"; + +let releaseRequest: (() => void) | undefined; +afterEach(() => { + releaseRequest?.(); + releaseRequest = undefined; + useSentryStore.getState().resetData(); + fetchContext.mockReset(); + vi.useRealTimers(); +}); + +test("a stalled context-lines request times out and releases later telemetry", async () => { + vi.useFakeTimers(); + useSentryStore.getState().resetData(); + let requestSignal: AbortSignal | undefined; + fetchContext.mockImplementation((_url: string, options: RequestInit) => { + requestSignal = options.signal ?? undefined; + return new Promise((_resolve, reject) => { + releaseRequest = () => reject(new Error("request released")); + requestSignal?.addEventListener("abort", () => reject(requestSignal?.reason), { once: true }); + }); + }); + const captured = useSentryStore.getState().ingestEnvelope([ + {}, + [ + [ + { type: "event" }, + { + event_id: "stalled-error", + timestamp: 1, + exception: { values: [{ type: "Error", value: "oops", stacktrace: { frames: [{ filename: "app.ts" }] } }] }, + }, + ], + ], + ]); + const later = useSentryStore + .getState() + .ingestEnvelope([ + {}, + [[{ type: "log" }, { items: [{ id: "later-log", body: "next", timestamp: 2, attributes: {} }] }]], + ]); + await vi.advanceTimersByTimeAsync(0); + expect(fetchContext).toHaveBeenCalledTimes(1); + expect(useSentryStore.getState().getLogs()).toEqual([]); + await vi.advanceTimersByTimeAsync(5000); + expect(requestSignal?.aborted).toBe(true); + await expect(captured).resolves.toMatchObject({ status: "processed" }); + await expect(later).resolves.toMatchObject({ status: "processed" }); + expect(useSentryStore.getState().getEventById("stalled-error")).toBeDefined(); + expect( + useSentryStore + .getState() + .getLogs() + .map(log => log.body), + ).toEqual(["next"]); +}); + +test("a failed exception context request still waits for successful sibling context", async () => { + useSentryStore.getState().resetData(); + let finishContext!: (response: unknown) => void; + fetchContext.mockRejectedValueOnce(new Error("context unavailable")).mockImplementationOnce( + () => + new Promise(resolve => { + finishContext = resolve; + releaseRequest = () => resolve({ status: 503 }); + }), + ); + const captured = useSentryStore.getState().ingestEnvelope([ + {}, + [ + [ + { type: "event" }, + { + event_id: "multiple-errors", + timestamp: 1, + exception: { + values: [ + { type: "Error", value: "first", stacktrace: { frames: [{ filename: "first.ts" }] } }, + { type: "Error", value: "second", stacktrace: { frames: [{ filename: "second.ts" }] } }, + ], + }, + }, + ], + ], + ]); + await vi.waitFor(() => expect(fetchContext).toHaveBeenCalledTimes(2)); + // The optional failure must not make the sibling's successful enrichment arrive after commit. + await new Promise(resolve => setTimeout(resolve, 0)); + expect(useSentryStore.getState().getEventById("multiple-errors")).toBeUndefined(); + finishContext({ + status: 200, + json: async () => ({ frames: [{ filename: "second.ts", context_line: "throw error;" }] }), + }); + await expect(captured).resolves.toMatchObject({ status: "processed" }); + expect(useSentryStore.getState().getEventById("multiple-errors")).toMatchObject({ + exception: { values: [{ value: "first" }, { stacktrace: { frames: [{ context_line: "throw error;" }] } }] }, + }); +}); diff --git a/packages/spotlight/src/ui/telemetry/store/helpers.ts b/packages/spotlight/src/ui/telemetry/store/helpers.ts index 3abd5c9ce..00ffd8f7d 100644 --- a/packages/spotlight/src/ui/telemetry/store/helpers.ts +++ b/packages/spotlight/src/ui/telemetry/store/helpers.ts @@ -1,6 +1 @@ -import type { Span } from "../types"; - -export function getAllSpansInTree(root: Span): Span[] { - const spans = [root]; - return root.children ? spans.concat(root.children.flatMap(getAllSpansInTree)) : spans; -} +export * from "../../../telemetry/store/helpers.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/slices/envelopesSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/envelopesSlice.ts index a1773cdb9..0fbf11b15 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/envelopesSlice.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/envelopesSlice.ts @@ -1,131 +1 @@ -import type { Envelope } from "@sentry/core"; -import { RAW_TYPES } from "@spotlight/shared/constants.ts"; -import type { StateCreator } from "zustand"; -import { ATTACHMENT_EVENT_TYPES, SUPPORTED_EVENT_TYPES } from "../../constants/sentry"; -import type { EventAttachment, Sdk, SentryEvent } from "../../types"; -import { sdkToPlatform } from "../../utils/sdkToPlatform"; -import type { EnvelopesSliceActions, EnvelopesSliceState, SentryStore } from "../types"; -import { normalizeSpanItem } from "../utils/spanProcessor"; - -const initialEnvelopesState: EnvelopesSliceState = { - envelopes: new Map(), -}; - -export const createEnvelopesSlice: StateCreator = ( - set, - get, -) => ({ - ...initialEnvelopesState, - pushEnvelope: (envelope: Envelope) => { - const [header, items] = envelope; - const lastSeen = new Date(header.sent_at as string).getTime(); - // Read the inferred source type from the envelope header (set by server) - const sourceType = (header as { __spotlight_inferred_source?: "browser" | "server" | "mobile" }) - .__spotlight_inferred_source; - let sdk: Sdk; - - if (header.sdk?.name && header.sdk.version) { - sdk = { - name: header.sdk.name, - version: header.sdk.version, - lastSeen: lastSeen, - }; - } else if (items.length > 0) { - sdk = get().inferSdkFromEvent(items[0][1] as SentryEvent); - } else { - sdk = { - name: "unknown", - version: "0.0.0", - lastSeen, - }; - } - - get().storeSdkRecord(sdk); - - const traceContext = header.trace; - - const attachmentsByEventId = new Map(); - const envelopeScopedAttachments: EventAttachment[] = []; - let eventCount = 0; - - for (const [itemHeader, itemData] of items) { - if (SUPPORTED_EVENT_TYPES.has(itemHeader.type)) { - eventCount += 1; - } - - if (!ATTACHMENT_EVENT_TYPES.has(itemHeader.type)) { - continue; - } - - const rawData = (itemData as { data?: unknown })?.data; - if (typeof rawData !== "string") { - continue; - } - - const attachment: EventAttachment = { - header: itemHeader, - data: rawData, - }; - - const eventId = "event_id" in itemHeader ? (itemHeader.event_id as string | undefined) : undefined; - - if (eventId) { - const existing = attachmentsByEventId.get(eventId) ?? []; - existing.push(attachment); - attachmentsByEventId.set(eventId, existing); - } else { - envelopeScopedAttachments.push(attachment); - } - } - - for (const [itemHeader, itemData] of items) { - if (itemHeader.type === "span") { - for (const event of normalizeSpanItem(itemData, sdk)) { - event.platform = sdkToPlatform(sdk.name); - if (sourceType) event.__sourceType = sourceType; - get().pushEvent(event); - } - continue; - } - if (SUPPORTED_EVENT_TYPES.has(itemHeader.type)) { - const item = itemData as SentryEvent; - if (!RAW_TYPES.has(itemHeader.type)) { - item.platform = sdkToPlatform(sdk.name); - } - if (traceContext) { - if (!item.contexts) { - item.contexts = {}; - } - item.contexts.trace ??= traceContext; - } - // Attach the inferred source type to the event for UI display - if (sourceType) { - (item as { __sourceType?: typeof sourceType }).__sourceType = sourceType; - } - const eventId = - item.event_id ?? ("event_id" in itemHeader ? (itemHeader.event_id as string | undefined) : undefined); - let attachmentsForEvent = eventId ? attachmentsByEventId.get(eventId) : undefined; - - if (!attachmentsForEvent?.length && eventCount === 1 && envelopeScopedAttachments.length > 0) { - attachmentsForEvent = envelopeScopedAttachments; - } - - if (attachmentsForEvent?.length) { - item.attachments = attachmentsForEvent; - } - // The below is an async function but we really don't need to wait for that - get().pushEvent(itemData as SentryEvent); - } - } - - const { envelopes } = get(); - const newEnvelopes = new Map(envelopes); - newEnvelopes.set(header.__spotlight_envelope_id as string, envelope); - set({ envelopes: newEnvelopes }); - return newEnvelopes.size; - }, - getEnvelopeById: (id: string) => { - return get().envelopes.get(id); - }, - getEnvelopes: () => Array.from(get().envelopes.values()), -}); +export * from "../../../../telemetry/store/slices/envelopesSlice.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/slices/eventsSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/eventsSlice.ts index 20bf682eb..a14a9e70d 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/eventsSlice.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/eventsSlice.ts @@ -1,213 +1 @@ -import { generateUuidv4 } from "@spotlight/ui/lib/uuid"; -import type { StateCreator } from "zustand"; -import { graftProfileSpans } from "../../data/profiles"; -import type { SentryEvent, SentryLogEventItem } from "../../types"; -import type { SentryTransactionEvent } from "../../types"; -import { isErrorEvent, isLogEvent, isProfileChunkEvent, isProfileEvent, isTraceEvent } from "../../utils/sentry"; -import type { EventsSliceActions, EventsSliceState, SentryProfileWithTraceMeta, SentryStore } from "../types"; -import { toTimestamp } from "../utils"; -import { processLogItems } from "../utils/logProcessor"; -import { mergeChunksToProfile, processProfileChunkEvent } from "../utils/profileChunkProcessor"; -import { processProfileEvent } from "../utils/profileProcessor"; -import { processSpanEvent } from "../utils/spanProcessor"; -import { initializeTrace } from "../utils/traceInitializer"; -import { processTransactionEvent, updateTraceMetadata } from "../utils/traceProcessor"; - -const initialEventsState: EventsSliceState = { - eventsById: new Map(), -}; - -export const createEventsSlice: StateCreator = ( - set, - get, -) => ({ - ...initialEventsState, - pushEvent: async (event: SentryEvent & { event_id?: string }) => { - if (!event.event_id) { - event.event_id = generateUuidv4(); - } - - const { eventsById: _eventsById } = get(); - if (_eventsById.has(event.event_id)) return; - - if (isErrorEvent(event)) { - await get().processStacktrace(event); - } - - if (event.timestamp) { - event.timestamp = toTimestamp(event.timestamp); - } - if (event.start_timestamp) { - event.start_timestamp = toTimestamp(event.start_timestamp); - } - - if (isLogEvent(event) && event.items?.length) { - const { logsById, logsByTraceId } = get(); - const existingLogIds = new Set(logsById.keys()); - const { processedLogs } = processLogItems(event, existingLogIds); - - // Update store with processed logs - // Build up the Maps incrementally to avoid stale state capture - const newLogsById = new Map(logsById); - const newLogsByTraceId = new Map(logsByTraceId); - - for (const logItem of processedLogs) { - newLogsById.set(logItem.id, logItem); - - if (logItem.trace_id) { - const logSet = newLogsByTraceId.get(logItem.trace_id) || new Set(); - logSet.add(logItem); - newLogsByTraceId.set(logItem.trace_id, logSet); - } - } - - // Set state once with all accumulated changes - set({ logsById: newLogsById, logsByTraceId: newLogsByTraceId }); - } - - const { eventsById } = get(); - const newEventIds = new Map(eventsById); - newEventIds.set(event.event_id, event); - set({ eventsById: newEventIds }); - - // Notify event subscribers - for (const [type, callback] of get().subscribers.values()) { - if (type === "event") { - (callback as (event: SentryEvent) => void)(event); - } - } - - const traceCtx = event.contexts?.trace; - if (traceCtx?.trace_id) { - const { tracesById, profilesByTraceId, profileChunksByProfilerId } = get(); - const existingTrace = tracesById.get(traceCtx.trace_id); - - // Initialize or get existing trace - let trace = existingTrace ?? initializeTrace(event); - - // Update timestamps for all events - trace.start_timestamp = Math.min(event.start_timestamp ?? event.timestamp, trace.start_timestamp); - trace.timestamp = Math.max(event.timestamp, trace.timestamp); - - let mergedV2Profile: SentryProfileWithTraceMeta | undefined; - - if (isTraceEvent(event)) { - // Process transaction event and update trace - const result = processTransactionEvent(event, { - existingTrace: trace, - profilesByTraceId, - profileChunksByProfilerId, - }); - trace = result.trace; - mergedV2Profile = result.mergedProfile; - } else if (event.type === "span") { - const result = processSpanEvent(event, { - existingTrace: trace, - profilesByTraceId, - profileChunksByProfilerId, - }); - trace = result.trace; - mergedV2Profile = result.mergedProfile; - } else if (isErrorEvent(event)) { - // For error events, increment error count - trace.errors += 1; - } - - // Update trace metadata and status for all events with trace context - updateTraceMetadata(trace); - - if (traceCtx.status !== "ok") { - trace.status = traceCtx.status; - } - - // Always save trace to store (whether new or updated) - const newTracesById = new Map(tracesById); - newTracesById.set(trace.trace_id, trace); - - // If we merged a V2 profile, store it by trace_id so the UI can find it - if (mergedV2Profile) { - const newProfilesByTraceId = new Map(profilesByTraceId); - newProfilesByTraceId.set(trace.trace_id, mergedV2Profile); - set({ tracesById: newTracesById, profilesByTraceId: newProfilesByTraceId }); - } else { - set({ tracesById: newTracesById }); - } - } - - if (isProfileEvent(event)) { - const { profilesByTraceId, tracesById } = get(); - const { profiles } = processProfileEvent(event, { tracesById }); - - // Update store with processed profiles - const newProfilesByTraceId = new Map(profilesByTraceId); - for (const { traceId, profile } of profiles) { - newProfilesByTraceId.set(traceId, profile); - } - set({ profilesByTraceId: newProfilesByTraceId }); - } - - // Handle V2 profile chunks (continuous profiling) - if (isProfileChunkEvent(event)) { - const { profileChunksByProfilerId, tracesById, eventsById, profilesByTraceId } = get(); - const { chunk } = processProfileChunkEvent(event); - - // Update store with processed profile chunk - const newProfileChunksByProfilerId = new Map(profileChunksByProfilerId); - const existingChunks = newProfileChunksByProfilerId.get(chunk.profiler_id) || []; - - // Check if this chunk already exists (by chunk_id) - if (!existingChunks.some(c => c.chunk_id === chunk.chunk_id)) { - const updatedChunks = [...existingChunks, chunk]; - newProfileChunksByProfilerId.set(chunk.profiler_id, updatedChunks); - - // Check if there's an existing transaction with this profiler_id that needs re-grafting - // This handles the case where profile chunks arrive AFTER the transaction - let traceToUpdate: string | undefined; - let activeThreadId: string | undefined; - for (const evt of eventsById.values()) { - if (isTraceEvent(evt)) { - const txn = evt as SentryTransactionEvent; - const profileCtx = txn.contexts?.profile as { profiler_id?: string } | undefined; - if (profileCtx?.profiler_id === chunk.profiler_id) { - traceToUpdate = txn.contexts.trace.trace_id; - activeThreadId = txn.contexts.trace.data?.["thread.id"] as string | undefined; - break; - } - } - } - - if (traceToUpdate) { - // Re-graft with updated chunks - const existingTrace = tracesById.get(traceToUpdate); - if (existingTrace && !existingTrace.profileGrafted) { - const mergedProfile = mergeChunksToProfile(updatedChunks, activeThreadId); - if (mergedProfile) { - // Create a shallow copy of the trace to trigger React re-render - const trace = { - ...existingTrace, - spans: new Map(existingTrace.spans), - spanTree: [...existingTrace.spanTree], - }; - graftProfileSpans(trace, mergedProfile); - - // Store updated trace and profile - const newTracesById = new Map(tracesById); - newTracesById.set(traceToUpdate, trace); - const newProfilesByTraceId = new Map(profilesByTraceId); - newProfilesByTraceId.set(traceToUpdate, mergedProfile); - set({ - profileChunksByProfilerId: newProfileChunksByProfilerId, - tracesById: newTracesById, - profilesByTraceId: newProfilesByTraceId, - }); - return; - } - } - } - - set({ profileChunksByProfilerId: newProfileChunksByProfilerId }); - } - } - }, - getEvents: () => Array.from(get().eventsById.values()), -}); +export * from "../../../../telemetry/store/slices/eventsSlice.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/slices/feedback.spec.ts b/packages/spotlight/src/ui/telemetry/store/slices/feedback.spec.ts index 1f39920d2..ad50842ed 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/feedback.spec.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/feedback.spec.ts @@ -8,7 +8,7 @@ describe("feedback envelope ingestion", () => { useSentryStore.getState().resetData(); }); - test("pushEnvelope stores a feedback item as a retrievable event", () => { + test("pushEnvelope stores a feedback item as a retrievable event", async () => { const envelope = [ { event_id: "feedback-event-id", @@ -35,7 +35,7 @@ describe("feedback envelope ingestion", () => { ], ] as unknown as Envelope; - useSentryStore.getState().pushEnvelope(envelope); + await useSentryStore.getState().pushEnvelope(envelope); const events = useSentryStore.getState().getEvents(); const feedbackEvents = events.filter(isFeedbackEvent); diff --git a/packages/spotlight/src/ui/telemetry/store/slices/logsSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/logsSlice.ts index 5b63e2a85..4c3cdf8c7 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/logsSlice.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/logsSlice.ts @@ -1,17 +1 @@ -import type { StateCreator } from "zustand"; -import type { LogsSliceActions, LogsSliceState, SentryStore } from "../types"; - -const initialLogsState: LogsSliceState = { - logsById: new Map(), - logsByTraceId: new Map(), -}; - -export const createLogsSlice: StateCreator = (_set, get) => ({ - ...initialLogsState, - getLogById: (id: string) => get().logsById.get(id), - getLogs: () => Array.from(get().logsById.values()), - getLogsByTraceId: (traceId: string) => { - const logsByTraceId = get().logsByTraceId.get(traceId); - return logsByTraceId ? Array.from(logsByTraceId) : []; - }, -}); +export * from "../../../../telemetry/store/slices/logsSlice.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/slices/profilesSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/profilesSlice.ts index 4a2f1dd52..096be6a6d 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/profilesSlice.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/profilesSlice.ts @@ -1,59 +1 @@ -import type { StateCreator } from "zustand"; -import type { AggregateCallData, TraceId } from "../../types"; -import { getFunctionNameFromFrame } from "../../utils/profileUtils"; -import type { ProfilesSliceActions, ProfilesSliceState, SentryStore } from "../types"; - -const initialProfilesState: ProfilesSliceState = { - profilesByTraceId: new Map(), - profileChunksByProfilerId: new Map(), -}; - -export const createProfilesSlice: StateCreator = ( - _set, - get, -) => ({ - ...initialProfilesState, - - // Unified profile access - works for both V1 and merged V2 profiles - getProfileByTraceId: (id: string) => get().profilesByTraceId.get(id), - - // Aggregate call data from all profiles (both V1 and V2 are stored in profilesByTraceId) - getAggregateCallData: () => { - const aggregateCalls = new Map(); - - // Process all profiles - unified storage for both V1 and merged V2 - for (const [traceId, profile] of get().profilesByTraceId) { - for (let sampleIdx = 0; sampleIdx < profile.samples.length - 1; sampleIdx++) { - const sample = profile.samples[sampleIdx]; - const nextSample = profile.samples[sampleIdx + 1]; - // TODO: Handle the case where nextSample is undefined -- use the end of the profile or associated trace - const duration = nextSample.start_timestamp - sample.start_timestamp; - // TODO: Keep a running average based on continuous samples - // as in where we keep seeing the same function name / frame back to back - - const stackId = sample.stack_id; - const frameIndices = profile.stacks[stackId]; - - for (const frameIdx of frameIndices) { - const frame = profile.frames[frameIdx]; - const name = getFunctionNameFromFrame(frame); - const callData = aggregateCalls.get(name); - if (callData) { - callData.totalTime += duration; - callData.samples += 1; - callData.traceIds.add(traceId); - } else { - aggregateCalls.set(name, { - name, - totalTime: duration, - samples: 1, - traceIds: new Set([traceId]), - }); - } - } - } - } - - return Array.from(aggregateCalls.values()); - }, -}); +export * from "../../../../telemetry/store/slices/profilesSlice.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/slices/sdksSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/sdksSlice.ts index afe81caed..78f61f532 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/sdksSlice.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/sdksSlice.ts @@ -1,42 +1 @@ -import type { StateCreator } from "zustand"; -import type { Sdk, SentryEvent } from "../../types"; -import type { SDKsSliceActions, SDKsSliceState, SentryStore } from "../types"; - -const initialSDKsState: SDKsSliceState = { - sdks: new Map(), -}; - -const getSDKKey = (sdk: Sdk): string => `${sdk.name}@${sdk.version}`; - -export const createSDKsSlice: StateCreator = (set, get) => ({ - ...initialSDKsState, - inferSdkFromEvent: (event: SentryEvent) => { - const sdk: Sdk = { - name: "unknown", - version: "unknown", - lastSeen: new Date().getTime(), - }; - - if (event.sdk) { - sdk.name = event.sdk.name || sdk.name; - sdk.version = event.sdk.version || sdk.version; - } else if (event.platform) { - sdk.name = event.platform; - } - - return sdk; - }, - storeSdkRecord: (sdk: Sdk) => { - const sdks = get().sdks; - const existingSdk = sdks.get(getSDKKey(sdk)); - if (existingSdk) { - existingSdk.lastSeen = sdk.lastSeen; - } else { - const newSdks = new Map(sdks); - newSdks.set(getSDKKey(sdk), sdk); - set({ sdks: newSdks }); - } - return existingSdk || sdk; - }, - getSdks: () => Array.from(get().sdks.values()), -}); +export * from "../../../../telemetry/store/slices/sdksSlice.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/slices/settingsSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/settingsSlice.ts deleted file mode 100644 index f7890b14e..000000000 --- a/packages/spotlight/src/ui/telemetry/store/slices/settingsSlice.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { CONTEXT_LINES_ENDPOINT } from "@spotlight/shared/constants.ts"; -import { DEFAULT_SIDECAR_URL } from "@spotlight/ui/constants"; -import type { StateCreator } from "zustand"; -import type { SentryStore, SettingsSliceActions, SettingsSliceState } from "../types"; - -const initialSettingsState: SettingsSliceState = { - contextLinesProvider: new URL(CONTEXT_LINES_ENDPOINT, DEFAULT_SIDECAR_URL).href, -}; - -export const createSettingsSlice: StateCreator< - SentryStore, - [], - [], - SettingsSliceState & SettingsSliceActions -> = set => ({ - ...initialSettingsState, - setSidecarUrl: (url: string) => { - const { href: contextLinesProviderUrl } = new URL(CONTEXT_LINES_ENDPOINT, url); - set({ contextLinesProvider: contextLinesProviderUrl }); - }, -}); diff --git a/packages/spotlight/src/ui/telemetry/store/slices/sharedSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/sharedSlice.ts index 29a8dedbf..5a23accf6 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/sharedSlice.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/sharedSlice.ts @@ -1,69 +1 @@ -import { log } from "@spotlight/ui/lib/logger"; -import type { SentryStore, SharedSliceActions } from "@spotlight/ui/telemetry/store/types"; -import type { SentryErrorEvent } from "@spotlight/ui/telemetry/types"; -import { getNativeFetchImplementation } from "@spotlight/ui/telemetry/utils/fetch"; -import type { StateCreator } from "zustand"; - -export const createSharedSlice: StateCreator = (set, get) => ({ - getEventById: (id: string) => get().eventsById.get(id), - getTraceById: (id: string) => get().tracesById.get(id), - getEventsByTrace: (traceId: string, spanId?: string | null) => { - const { getEvents } = get(); - return getEvents().filter(evt => { - const trace = evt.contexts?.trace; - if (!trace || trace.trace_id !== traceId) return false; - if (spanId !== undefined) return trace.span_id === spanId; - return true; - }); - }, - processStacktrace: async (errorEvent: SentryErrorEvent): Promise => { - if (!errorEvent.exception || !errorEvent.exception.values) { - return; - } - - await Promise.all( - (errorEvent.exception.values ?? []).map(async exception => { - if (!exception.stacktrace || !exception.stacktrace.frames) { - return; - } - exception.stacktrace.frames.reverse(); - - if (exception.stacktrace.frames.every(frame => frame.post_context && frame.pre_context && frame.context_line)) { - log("Skipping contextlines request as we have full context for", exception); - return; - } - - try { - const makeFetch = getNativeFetchImplementation(); - const stackTraceWithContextResponse = await makeFetch(get().contextLinesProvider, { - method: "PUT", - body: JSON.stringify(exception.stacktrace), - }); - - if (!stackTraceWithContextResponse.ok || stackTraceWithContextResponse.status !== 200) { - return; - } - - const stackTraceWithContext = await stackTraceWithContextResponse.json(); - exception.stacktrace = stackTraceWithContext; - } catch { - // Something went wrong, for now we just ignore it. - } - - return; - }), - ); - }, - resetData: () => { - set({ - envelopes: new Map(), - eventsById: new Map(), - tracesById: new Map(), - sdks: new Map(), - profilesByTraceId: new Map(), - profileChunksByProfilerId: new Map(), - logsById: new Map(), - logsByTraceId: new Map(), - }); - }, -}); +export * from "../../../../telemetry/store/slices/sharedSlice.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/slices/spans.spec.ts b/packages/spotlight/src/ui/telemetry/store/slices/spans.spec.ts index 924247336..c8a3888d1 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/spans.spec.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/spans.spec.ts @@ -44,7 +44,7 @@ function segment(overrides: Record = {}) { }); } -function ingest(type: string, payload: unknown, contentType?: string): Envelope { +async function ingest(type: string, payload: unknown, contentType?: string): Promise { const parsed = processEnvelope({ contentType: "application/x-sentry-envelope", data: [ @@ -54,11 +54,11 @@ function ingest(type: string, payload: unknown, contentType?: string): Envelope ].join("\n"), }); expect(parsed).not.toBeNull(); - useSentryStore.getState().pushEnvelope(parsed!.envelope); + await useSentryStore.getState().pushEnvelope(parsed!.envelope); return parsed!.envelope; } -function ingestSpans(items: unknown[]) { +async function ingestSpans(items: unknown[]) { return ingest("span", { version: 2, items }, "application/vnd.sentry.items.span.v2+json"); } @@ -99,8 +99,8 @@ function legacyTransaction() { describe("standalone span envelope ingestion", () => { beforeEach(() => useSentryStore.getState().resetData()); - test("v2 database spans populate Queries with names, typed attributes and millisecond timing", () => { - const envelope = ingestSpans([streamedSpan(), segment()]); + test("v2 database spans populate Queries with names, typed attributes and millisecond timing", async () => { + const envelope = await ingestSpans([streamedSpan(), segment()]); expect(useSentryStore.getState().getTraces()).toHaveLength(1); expect(getQueries()).toHaveLength(1); @@ -120,14 +120,14 @@ describe("standalone span envelope ingestion", () => { expect(envelope[1][0][1]).toMatchObject({ version: 2, items: [{ name: query }, {}] }); }); - test("queries are visible before their segment ends and late parents replace orphan placeholders", () => { + test("queries are visible before their segment ends and late parents replace orphan placeholders", async () => { const parentId = "83aed660f800afd4"; - ingestSpans([streamedSpan({ parent_span_id: parentId })]); + await ingestSpans([streamedSpan({ parent_span_id: parentId })]); expect(getQueries()).toHaveLength(1); - ingestSpans([segment()]); - ingestSpans([streamedSpan({ span_id: parentId, name: "prisma query", attributes: {} })]); - ingestSpans([streamedSpan({ span_id: "bbbbbbbbbbbbbbbb", parent_span_id: parentId })]); + await ingestSpans([segment()]); + await ingestSpans([streamedSpan({ span_id: parentId, name: "prisma query", attributes: {} })]); + await ingestSpans([streamedSpan({ span_id: "bbbbbbbbbbbbbbbb", parent_span_id: parentId })]); expect(getQueries()).toHaveLength(2); expect(getTrace().transactions).toHaveLength(1); @@ -137,18 +137,18 @@ describe("standalone span envelope ingestion", () => { expect(getTrace().timestamp - getTrace().start_timestamp).toBe(1000); }); - test("repeated delivery does not duplicate query counts or span tree children", () => { - ingestSpans([segment(), streamedSpan()]); - ingestSpans([streamedSpan(), segment()]); + test("repeated delivery does not duplicate query counts or span tree children", async () => { + await ingestSpans([segment(), streamedSpan()]); + await ingestSpans([streamedSpan(), segment()]); expect(getQueries()).toHaveLength(1); expect(treeIds(getTrace().spanTree)).toEqual([rootId, queryId]); expect(getTrace().transactions).toHaveLength(1); }); - test.each([true, false])("legacy transactions and streamed queries coexist (transaction first: %s)", first => { - if (first) ingest("transaction", legacyTransaction()); - ingestSpans([streamedSpan()]); - if (!first) ingest("transaction", legacyTransaction()); + test.each([true, false])("legacy transactions and streamed queries coexist (transaction first: %s)", async first => { + if (first) await ingest("transaction", legacyTransaction()); + await ingestSpans([streamedSpan()]); + if (!first) await ingest("transaction", legacyTransaction()); expect( getQueries() @@ -158,8 +158,8 @@ describe("standalone span envelope ingestion", () => { expect(treeIds(getTrace().spanTree).sort()).toEqual([rootId, queryId, "aaaaaaaaaaaaaaaa"].sort()); }); - test("legacy standalone span items are accepted alongside v2 containers", () => { - ingest("span", { + test("legacy standalone span items are accepted alongside v2 containers", async () => { + await ingest("span", { trace_id: traceId, span_id: queryId, parent_span_id: rootId, @@ -169,34 +169,40 @@ describe("standalone span envelope ingestion", () => { timestamp: 1793000000.25, data: { "db.system": "postgresql" }, }); - ingestSpans([segment()]); + await ingestSpans([segment()]); expect(getQueries()[0]).toMatchObject({ description: query, data: { "db.system": "postgresql" } }); expect(treeIds(getTrace().spanTree)).toEqual([rootId, queryId]); }); - test("segment spans with remote parents still supply the local trace name", () => { - ingestSpans([streamedSpan(), segment({ parent_span_id: "cccccccccccccccc" })]); + test("segment spans with remote parents still supply the local trace name", async () => { + await ingestSpans([streamedSpan(), segment({ parent_span_id: "cccccccccccccccc" })]); expect(getTrace().rootTransactionName).toBe("GET /applications/[applicationId]/subscriptions"); expect(getTrace().rootTransaction?.contexts.trace.span_id).toBe(rootId); expect(getQueries()).toHaveLength(1); }); - test("unsupported containers and invalid spans do not prevent valid spans from being ingested", () => { - ingest("span", { version: 99, items: [streamedSpan()] }); + test("unsupported containers and invalid spans do not prevent valid spans from being ingested", async () => { + await ingest("span", { version: 99, items: [streamedSpan()] }); expect(useSentryStore.getState().getTraces()).toHaveLength(0); - ingestSpans([null, {}, streamedSpan({ end_timestamp: undefined }), streamedSpan({ trace_id: "" }), streamedSpan()]); + await ingestSpans([ + null, + {}, + streamedSpan({ end_timestamp: undefined }), + streamedSpan({ trace_id: "" }), + streamedSpan(), + ]); expect(getQueries()).toHaveLength(1); expect(Number.isFinite(getTrace().timestamp)).toBe(true); }); - test("span IDs are scoped to their trace", () => { - ingestSpans([streamedSpan(), streamedSpan({ trace_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" })]); + test("span IDs are scoped to their trace", async () => { + await ingestSpans([streamedSpan(), streamedSpan({ trace_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" })]); expect(useSentryStore.getState().getTraces()).toHaveLength(2); expect(getQueries()).toHaveLength(1); }); - test("segment attributes survive subsequent segments in the same distributed trace", () => { - ingestSpans([ + test("segment attributes survive subsequent segments in the same distributed trace", async () => { + await ingestSpans([ segment({ attributes: { "sentry.op": { type: "string", value: "http.server" }, @@ -205,7 +211,7 @@ describe("standalone span envelope ingestion", () => { }, }), ]); - ingestSpans([segment({ span_id: "dddddddddddddddd", parent_span_id: rootId })]); + await ingestSpans([segment({ span_id: "dddddddddddddddd", parent_span_id: rootId })]); expect(getTrace().spans.get(rootId)?.data).toMatchObject({ "http.request.method": "GET", "http.response.status_code": 200, diff --git a/packages/spotlight/src/ui/telemetry/store/slices/subscriptionsSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/subscriptionsSlice.ts index 1f7fa39d1..7859529b2 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/subscriptionsSlice.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/subscriptionsSlice.ts @@ -1,30 +1 @@ -import { generateUuidv4 } from "@spotlight/ui/lib/uuid"; -import type { StateCreator } from "zustand"; -import type { SentryStore, Subscription, SubscriptionsSliceActions, SubscriptionsSliceState } from "../types"; - -const initialSubscriptionsState: SubscriptionsSliceState = { - subscribers: new Map(), -}; - -export const createSubscriptionsSlice: StateCreator< - SentryStore, - [], - [], - SubscriptionsSliceState & SubscriptionsSliceActions -> = (set, get) => ({ - ...initialSubscriptionsState, - subscribe: (...args: Subscription) => { - const id = generateUuidv4(); - const { subscribers } = get(); - const newSubscribers = new Map(subscribers); - newSubscribers.set(id, args); - set({ subscribers: newSubscribers }); - - return () => { - const { subscribers: currentSubscribers } = get(); - const updatedSubscribers = new Map(currentSubscribers); - updatedSubscribers.delete(id); - set({ subscribers: updatedSubscribers }); - }; - }, -}); +export * from "../../../../telemetry/store/slices/subscriptionsSlice.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/slices/tracesSlice.ts b/packages/spotlight/src/ui/telemetry/store/slices/tracesSlice.ts index b2b79f35a..d0325f274 100644 --- a/packages/spotlight/src/ui/telemetry/store/slices/tracesSlice.ts +++ b/packages/spotlight/src/ui/telemetry/store/slices/tracesSlice.ts @@ -1,14 +1 @@ -import type { StateCreator } from "zustand"; -import type { SentryStore, TracesSliceActions, TracesSliceState } from "../types"; - -const initialTracesState: TracesSliceState = { - tracesById: new Map(), -}; - -export const createTracesSlice: StateCreator = ( - _, - get, -) => ({ - ...initialTracesState, - getTraces: () => Array.from(get().tracesById.values()), -}); +export * from "../../../../telemetry/store/slices/tracesSlice.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/store.ts b/packages/spotlight/src/ui/telemetry/store/store.ts index da5c6f0ce..a2ead4830 100644 --- a/packages/spotlight/src/ui/telemetry/store/store.ts +++ b/packages/spotlight/src/ui/telemetry/store/store.ts @@ -1,25 +1,150 @@ -import { create } from "zustand"; -import { createEnvelopesSlice } from "./slices/envelopesSlice"; -import { createEventsSlice } from "./slices/eventsSlice"; -import { createLogsSlice } from "./slices/logsSlice"; -import { createProfilesSlice } from "./slices/profilesSlice"; -import { createSDKsSlice } from "./slices/sdksSlice"; -import { createSettingsSlice } from "./slices/settingsSlice"; -import { createSharedSlice } from "./slices/sharedSlice"; -import { createSubscriptionsSlice } from "./slices/subscriptionsSlice"; -import { createTracesSlice } from "./slices/tracesSlice"; -import type { SentryStore, SharedSliceActions } from "./types"; +import type { Envelope } from "@sentry/core"; +import { CONTEXT_LINES_ENDPOINT } from "@spotlight/shared/constants.ts"; +import { DEFAULT_SIDECAR_URL } from "@spotlight/ui/constants"; +import { useStore } from "zustand"; +import { createTelemetry } from "../../../sdk/core.ts"; +import type { IngestResult, TelemetrySnapshot } from "../../../sdk/core.ts"; +import type { SentryStore, SentryStoreState, Subscription } from "../../../telemetry/store/types.ts"; +import { getNativeFetchImplementation } from "../utils/fetch"; -const useSentryStore = create()((...a) => ({ - ...createEventsSlice(...a), - ...createTracesSlice(...a), - ...createProfilesSlice(...a), - ...createSubscriptionsSlice(...a), - ...createSettingsSlice(...a), - ...createEnvelopesSlice(...a), - ...createSDKsSlice(...a), - ...createLogsSlice(...a), - ...createSharedSlice(...a), -})); +let contextLinesProvider = new URL(CONTEXT_LINES_ENDPOINT, DEFAULT_SIDECAR_URL).href; +export function setTelemetrySidecarUrl(url: string) { + contextLinesProvider = new URL(CONTEXT_LINES_ENDPOINT, url).href; +} +const spotlight = createTelemetry({ + enrichStacktrace: async event => { + await Promise.all( + (event.exception?.values ?? []).map(async exception => { + const frames = exception.stacktrace?.frames; + if (!frames || frames.every(frame => frame.post_context && frame.pre_context && frame.context_line)) return; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + try { + const response = await getNativeFetchImplementation()(contextLinesProvider, { + method: "PUT", + body: JSON.stringify(exception.stacktrace), + signal: controller.signal, + }); + if (response.status === 200) exception.stacktrace = await response.json(); + } catch { + // Optional context must not prevent sibling exceptions from finishing enrichment. + } finally { + clearTimeout(timeout); + } + }), + ); + }, +}); + +type UiState = SentryStoreState & + Pick< + SentryStore, + | "getEventById" + | "getTraceById" + | "getEventsByTrace" + | "getEvents" + | "getTraces" + | "getLogs" + | "getLogById" + | "getLogsByTraceId" + | "getProfileByTraceId" + | "getAggregateCallData" + | "getEnvelopeById" + | "getEnvelopes" + | "getSdks" + | "resetData" + | "subscribe" + > & { + pushEnvelope: (input: unknown) => Promise; + ingestEnvelope: (input: unknown) => Promise; + }; + +// Only this adapter translates the historical UI vocabulary. The SDK never exposes these actions. +const subscribers = new Map(); +const actions = { + getEventById: spotlight.getEvent, + getTraceById: spotlight.getTrace, + getEventsByTrace: (traceId: string, spanId?: string | null) => [ + ...spotlight.getEvents({ traceId, ...(spanId == null ? {} : { spanId }) }), + ], + getEvents: () => [...spotlight.getEvents()], + getTraces: () => [...spotlight.getTraces()], + getLogs: () => [...spotlight.getLogs()], + getLogById: spotlight.getLog, + getLogsByTraceId: (traceId: string) => [...spotlight.getLogs({ traceId })], + getProfileByTraceId: spotlight.getProfile, + getAggregateCallData: () => + spotlight.getProfileFunctionSummaries().map(summary => ({ + name: summary.name, + totalTime: summary.totalDurationMs, + samples: summary.sampleCount, + traceIds: new Set(summary.traceIds), + })), + getEnvelopeById: spotlight.getEnvelope, + getEnvelopes: () => [...spotlight.getEnvelopes()] as Envelope[], + getSdks: () => [...spotlight.getSdks()], + resetData: spotlight.reset, + pushEnvelope: spotlight.ingestEnvelope, + ingestEnvelope: spotlight.ingestEnvelope, + subscribe: (...subscription: Subscription) => { + const id = crypto.randomUUID(); + subscribers.set(id, subscription); + return () => { + subscribers.delete(id); + }; + }, +}; +function view(snapshot: TelemetrySnapshot): UiState { + const logsByTraceId = new Map>(); + for (const log of snapshot.logs) { + if (!log.trace_id) continue; + const group = logsByTraceId.get(log.trace_id) ?? new Set(); + group.add(log); + logsByTraceId.set(log.trace_id, group); + } + return { + eventsById: new Map(snapshot.events.map(event => [event.event_id, event])), + tracesById: new Map(snapshot.traces.map(trace => [trace.trace_id, trace])), + logsById: new Map(snapshot.logs.map(log => [log.id, log])), + logsByTraceId, + envelopes: new Map(snapshot.envelopes.map(envelope => [String(envelope[0].__spotlight_envelope_id), envelope])), + sdks: new Map(snapshot.sdks.map(sdk => [`${sdk.name}@${sdk.version}`, sdk])), + profilesByTraceId: new Map(snapshot.profiles), + profileChunksByProfilerId: new Map(), + subscribers, + ...actions, + } as unknown as UiState; +} +let current = view(spotlight.getSnapshot()); +const initial = current; +const listeners = new Set<(state: UiState, previous: UiState) => void>(); +spotlight.subscribe(snapshot => { + const previous = current; + current = view(snapshot); + for (const listener of listeners) listener(current, previous); + for (const event of snapshot.events) { + if (event.event_id && previous.eventsById.has(event.event_id)) continue; + for (const [type, callback] of subscribers.values()) { + if (type === "event") callback(event as Parameters[0]); + if (spotlight.getSnapshot() !== snapshot) return; + } + } +}); +const adapter = { + getState: () => current, + getInitialState: () => initial, + subscribe: (listener: (state: UiState, previous: UiState) => void) => { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, +}; +function useTelemetryStore( + selector: (state: UiState) => T = (state => state) as (state: UiState) => T, +): T { + return useStore(adapter, selector); +} +const useSentryStore = Object.assign(useTelemetryStore, adapter); export default useSentryStore; diff --git a/packages/spotlight/src/ui/telemetry/store/types.ts b/packages/spotlight/src/ui/telemetry/store/types.ts index b0159a92d..8dc71445f 100644 --- a/packages/spotlight/src/ui/telemetry/store/types.ts +++ b/packages/spotlight/src/ui/telemetry/store/types.ts @@ -1,127 +1 @@ -import type { Envelope } from "@sentry/core"; -import type { - AggregateCallData, - Sdk, - SentryErrorEvent, - SentryEvent, - SentryLogEventItem, - SentryProcessedProfile, - Trace, -} from "../types"; -import type { ProcessedProfileChunk } from "./utils/profileChunkProcessor"; - -export type SentryProfileWithTraceMeta = SentryProcessedProfile & { - timestamp: number; - active_thread_id: string; -}; - -export type OnlineSubscription = ["online", (status: boolean) => void]; -export type EventSubscription = ["event", (event: SentryEvent) => void]; -export type TraceSubscription = ["trace", (trace: Trace) => void]; -export type Subscription = OnlineSubscription | EventSubscription | TraceSubscription; - -export interface EventsSliceState { - eventsById: Map; -} - -export interface EventsSliceActions { - pushEvent: (event: SentryEvent & { event_id?: string }) => Promise; - getEvents: () => SentryEvent[]; -} - -export interface TracesSliceState { - tracesById: Map; -} - -export interface TracesSliceActions { - getTraces: () => Trace[]; -} - -export interface ProfilesSliceState { - /** Final merged profiles keyed by trace_id - works for both V1 and V2 */ - profilesByTraceId: Map; - /** Internal: V2 chunks being accumulated, keyed by profiler_id */ - profileChunksByProfilerId: Map; -} - -export interface ProfilesSliceActions { - /** Get profile for a trace - unified access for both V1 and V2 profiles */ - getProfileByTraceId: (id: string) => SentryProfileWithTraceMeta | undefined; - getAggregateCallData: () => AggregateCallData[]; -} - -export interface SubscriptionsSliceState { - subscribers: Map; -} - -export interface SubscriptionsSliceActions { - subscribe: (...args: Subscription) => () => void; -} - -export interface SettingsSliceState { - contextLinesProvider: string; -} - -export interface SettingsSliceActions { - setSidecarUrl: (url: string) => void; -} - -export interface EnvelopesSliceState { - envelopes: Map; -} - -export interface EnvelopesSliceActions { - pushEnvelope: (params: Envelope) => number; - getEnvelopeById: (id: string) => Envelope | undefined; - getEnvelopes: () => Array; -} - -export interface SDKsSliceState { - sdks: Map; -} - -export interface LogsSliceState { - logsById: Map; - logsByTraceId: Map>; -} - -export interface LogsSliceActions { - getLogById: (id: string) => SentryLogEventItem | undefined; - getLogs: () => SentryLogEventItem[]; - getLogsByTraceId: (traceId: string) => SentryLogEventItem[]; -} - -export interface SDKsSliceActions { - inferSdkFromEvent: (event: SentryEvent) => Sdk; - storeSdkRecord: (sdk: Sdk) => Sdk; - getSdks: () => Sdk[]; -} - -export interface SharedSliceActions { - getEventById: (id: string) => SentryEvent | undefined; - getTraceById: (id: string) => Trace | undefined; - getEventsByTrace: (traceId: string, spanId?: string | null) => SentryEvent[]; - processStacktrace: (errorEvent: SentryErrorEvent) => Promise; - resetData: () => void; -} - -export type SentryStoreState = EventsSliceState & - TracesSliceState & - ProfilesSliceState & - SubscriptionsSliceState & - SettingsSliceState & - EnvelopesSliceState & - LogsSliceState & - SDKsSliceState; - -export type SentryStoreActions = EventsSliceActions & - TracesSliceActions & - ProfilesSliceActions & - SubscriptionsSliceActions & - SettingsSliceActions & - EnvelopesSliceActions & - LogsSliceActions & - SDKsSliceActions & - SharedSliceActions; - -export type SentryStore = SentryStoreState & SentryStoreActions; +export * from "../../../telemetry/store/types.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/utils.ts b/packages/spotlight/src/ui/telemetry/store/utils.ts index 926c9c536..dff81aba6 100644 --- a/packages/spotlight/src/ui/telemetry/store/utils.ts +++ b/packages/spotlight/src/ui/telemetry/store/utils.ts @@ -1,9 +1 @@ -export function toTimestamp(date: string | number) { - if (typeof date === "string") return new Date(date).getTime(); - return date * 1000; -} - -export function relativeNsToTimestamp(startTs: number, ns: number | string) { - const nsStr = ns.toString(); - return nsStr.length > 3 ? startTs + Number.parseInt(nsStr.slice(0, -3), 10) / 1000 : startTs; -} +export * from "../../../telemetry/store/utils.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/utils/logProcessor.ts b/packages/spotlight/src/ui/telemetry/store/utils/logProcessor.ts index cfe645f6e..9406f5cfe 100644 --- a/packages/spotlight/src/ui/telemetry/store/utils/logProcessor.ts +++ b/packages/spotlight/src/ui/telemetry/store/utils/logProcessor.ts @@ -1,46 +1 @@ -import { generateUuidv4 } from "@spotlight/ui/lib/uuid"; -import type { SentryLogEvent, SentryLogEventItem } from "../../types"; -import { toTimestamp } from "../utils"; - -export interface LogProcessingResult { - processedLogs: SentryLogEventItem[]; -} - -/** - * Processes log items from a log event, normalizing and enriching them with metadata. - * @param event The log event to process - * @param existingLogIds Set of already processed log IDs to avoid duplicates - * @returns Processed log items ready to be stored - */ -export function processLogItems(event: SentryLogEvent, existingLogIds: Set): LogProcessingResult { - const processedLogs: SentryLogEventItem[] = []; - - if (!event.items?.length) { - return { processedLogs }; - } - - for (const logItem of event.items) { - const logId = logItem.id || generateUuidv4(); - - // Skip if already processed - if (existingLogIds.has(logId)) { - continue; - } - - // Normalize severity number - if (logItem.severity_number == null) { - logItem.severity_number = 0; - } - - // Extract SDK name from attributes - logItem.sdk = logItem.attributes?.["sentry.sdk.name"]?.value as string; - - // Normalize timestamp - logItem.timestamp = toTimestamp(logItem.timestamp); - logItem.id = logId; - - processedLogs.push(logItem); - } - - return { processedLogs }; -} +export * from "../../../../telemetry/store/utils/logProcessor.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/utils/profileChunkProcessor.ts b/packages/spotlight/src/ui/telemetry/store/utils/profileChunkProcessor.ts index 7a5272fe1..f7b100e41 100644 --- a/packages/spotlight/src/ui/telemetry/store/utils/profileChunkProcessor.ts +++ b/packages/spotlight/src/ui/telemetry/store/utils/profileChunkProcessor.ts @@ -1,157 +1 @@ -import type { ProcessedProfileSample, SentryProfileV2ChunkEvent } from "../../types"; -import type { SentryProfileWithTraceMeta } from "../types"; - -/** - * Represents a processed V2 profile chunk ready for storage and visualization. - * V2 chunks use absolute timestamps and are keyed by profiler_id. - */ -export type ProcessedProfileChunk = { - profiler_id: string; - chunk_id: string; - platform: string; - release?: string; - environment?: string; - samples: ProcessedProfileSample[]; - stacks: number[][]; - frames: SentryProfileV2ChunkEvent["profile"]["frames"]; - thread_metadata: SentryProfileV2ChunkEvent["profile"]["thread_metadata"]; - // Timestamps derived from samples - start_timestamp: number; - end_timestamp: number; -}; - -export interface ProfileChunkProcessingResult { - chunk: ProcessedProfileChunk; -} - -/** - * Processes a V2 profile chunk event (continuous profiling). - * V2 samples use absolute Unix timestamps, so no relative-to-start conversion needed. - */ -export function processProfileChunkEvent(event: SentryProfileV2ChunkEvent): ProfileChunkProcessingResult { - const { profile, profiler_id, chunk_id, platform, release, environment } = event; - - // Convert V2 samples to processed format - // V2 uses absolute Unix timestamps in seconds with microsecond precision - const processedSamples: ProcessedProfileSample[] = profile.samples.map(sample => ({ - stack_id: sample.stack_id, - thread_id: sample.thread_id, - start_timestamp: sample.timestamp * 1000, // Convert to milliseconds for internal use - })); - - // Sort samples by timestamp - processedSamples.sort((a, b) => a.start_timestamp - b.start_timestamp); - - // Calculate chunk time boundaries from samples - const start_timestamp = processedSamples.length > 0 ? processedSamples[0].start_timestamp : Date.now(); - const end_timestamp = - processedSamples.length > 0 ? processedSamples[processedSamples.length - 1].start_timestamp : start_timestamp; - - // Reverse stacks to match the order expected by visualization (leaf first) - const reversedStacks = profile.stacks.map((s: number[]) => Array.from(s).reverse()); - - const chunk: ProcessedProfileChunk = { - profiler_id, - chunk_id, - platform, - release, - environment, - samples: processedSamples, - stacks: reversedStacks, - frames: profile.frames, - thread_metadata: profile.thread_metadata, - start_timestamp, - end_timestamp, - }; - - return { chunk }; -} - -/** - * Merges multiple profile chunks from the same profiler session into a single profile - * that can be used with existing profile visualization components. - */ -export function mergeChunksToProfile( - chunks: ProcessedProfileChunk[], - activeThreadId?: string, -): SentryProfileWithTraceMeta | null { - if (chunks.length === 0) return null; - - // Sort chunks by start timestamp - const sortedChunks = [...chunks].sort((a, b) => a.start_timestamp - b.start_timestamp); - - // Merge frames and build frame index mapping - const mergedFrames: ProcessedProfileChunk["frames"] = []; - const frameIndexMaps: Map[] = []; - - for (const chunk of sortedChunks) { - const frameMap = new Map(); - for (let i = 0; i < chunk.frames.length; i++) { - // For simplicity, we add all frames and dedupe by reference - // A more sophisticated approach would dedupe by frame content - const newIndex = mergedFrames.length; - mergedFrames.push(chunk.frames[i]); - frameMap.set(i, newIndex); - } - frameIndexMaps.push(frameMap); - } - - // Merge stacks with remapped frame indices - const mergedStacks: number[][] = []; - const stackIndexMaps: Map[] = []; - - for (let chunkIdx = 0; chunkIdx < sortedChunks.length; chunkIdx++) { - const chunk = sortedChunks[chunkIdx]; - const frameMap = frameIndexMaps[chunkIdx]; - const stackMap = new Map(); - - for (let i = 0; i < chunk.stacks.length; i++) { - const remappedStack = chunk.stacks[i].map(frameIdx => frameMap.get(frameIdx) ?? frameIdx); - const newIndex = mergedStacks.length; - mergedStacks.push(remappedStack); - stackMap.set(i, newIndex); - } - stackIndexMaps.push(stackMap); - } - - // Merge samples with remapped stack indices - const mergedSamples: ProcessedProfileSample[] = []; - for (let chunkIdx = 0; chunkIdx < sortedChunks.length; chunkIdx++) { - const chunk = sortedChunks[chunkIdx]; - const stackMap = stackIndexMaps[chunkIdx]; - - for (const sample of chunk.samples) { - mergedSamples.push({ - ...sample, - stack_id: stackMap.get(sample.stack_id) ?? sample.stack_id, - }); - } - } - - // Sort all samples by timestamp - mergedSamples.sort((a, b) => a.start_timestamp - b.start_timestamp); - - // Merge thread metadata - const mergedThreadMetadata: ProcessedProfileChunk["thread_metadata"] = {}; - for (const chunk of sortedChunks) { - Object.assign(mergedThreadMetadata, chunk.thread_metadata); - } - - // Determine active thread ID - use provided or first thread with samples - const resolvedActiveThreadId = - activeThreadId || - (mergedSamples.length > 0 ? mergedSamples[0].thread_id : Object.keys(mergedThreadMetadata)[0] || "0"); - - // Calculate overall timestamp from first chunk - const timestamp = sortedChunks[0].start_timestamp; - - return { - platform: sortedChunks[0].platform, - thread_metadata: mergedThreadMetadata, - samples: mergedSamples, - frames: mergedFrames, - stacks: mergedStacks, - timestamp, - active_thread_id: resolvedActiveThreadId, - }; -} +export * from "../../../../telemetry/store/utils/profileChunkProcessor.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/utils/profileProcessor.ts b/packages/spotlight/src/ui/telemetry/store/utils/profileProcessor.ts index 625fe9dc5..d428bcee2 100644 --- a/packages/spotlight/src/ui/telemetry/store/utils/profileProcessor.ts +++ b/packages/spotlight/src/ui/telemetry/store/utils/profileProcessor.ts @@ -1,75 +1 @@ -import { graftProfileSpans } from "../../data/profiles"; -import type { ProfileSample, SentryProfileTransactionInfo, SentryProfileV1Event, Trace } from "../../types"; -import type { SentryProfileWithTraceMeta } from "../types"; -import { relativeNsToTimestamp } from "../utils"; - -export interface ProfileProcessingContext { - tracesById: Map; -} - -export interface ProfileProcessingResult { - profiles: Array<{ - traceId: string; - profile: SentryProfileWithTraceMeta; - }>; -} - -/** - * Processes a profile event and creates profile data for each transaction. - * Handles profile grafting onto existing traces if available. - */ -export function processProfileEvent( - event: SentryProfileV1Event, - context: ProfileProcessingContext, -): ProfileProcessingResult { - const { tracesById } = context; - const profiles: ProfileProcessingResult["profiles"] = []; - - // Ensure transactions array exists - if (!event.transactions) { - event.transactions = event.transaction ? [event.transaction] : []; - } - - for (const txn of event.transactions) { - // Skip string transaction IDs - if (typeof txn === "string") continue; - - const profileTxn = txn as SentryProfileTransactionInfo; - const trace = tracesById.get(profileTxn.trace_id); - - // Calculate timestamp - const timestamp = - trace && profileTxn.relative_start_ns != null - ? relativeNsToTimestamp(trace.start_timestamp, profileTxn.relative_start_ns) - : event.timestamp; - - // Build profile data - const profile: SentryProfileWithTraceMeta = { - platform: event.platform, - thread_metadata: event.profile.thread_metadata, - samples: event.profile.samples.map((s: ProfileSample) => ({ - stack_id: s.stack_id, - thread_id: s.thread_id, - elapsed_since_start_ns: s.elapsed_since_start_ns, - start_timestamp: relativeNsToTimestamp(timestamp, s.elapsed_since_start_ns), - })), - frames: event.profile.frames, - stacks: event.profile.stacks.map((s: number[]) => Array.from(s).reverse()), - timestamp, - active_thread_id: profileTxn.active_thread_id, - }; - - profiles.push({ - traceId: profileTxn.trace_id, - profile, - }); - - // Graft profile onto trace if it's ready - // Avoid grafting partial traces (where we mocked start_timestamp from event timestamp) - if (trace && trace.start_timestamp < trace.timestamp) { - graftProfileSpans(trace, profile); - } - } - - return { profiles }; -} +export * from "../../../../telemetry/store/utils/profileProcessor.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/utils/spanProcessor.ts b/packages/spotlight/src/ui/telemetry/store/utils/spanProcessor.ts index e61ff2e78..d8efc3b6c 100644 --- a/packages/spotlight/src/ui/telemetry/store/utils/spanProcessor.ts +++ b/packages/spotlight/src/ui/telemetry/store/utils/spanProcessor.ts @@ -1,107 +1 @@ -import type { Sdk, SentrySpanEvent, SentryTransactionEvent, Span } from "../../types"; -import { toTimestamp } from "../utils"; -import { initializeTrace } from "./traceInitializer"; -import { - type TraceProcessingContext, - type TraceProcessingResult, - processTransactionEvent, - rebuildSpanTree, -} from "./traceProcessor"; - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -/** Decode the span v2 container, or a legacy standalone span, without changing the raw envelope. */ -export function normalizeSpanItem(payload: unknown, sdk: Sdk): SentrySpanEvent[] { - if (!isRecord(payload)) return []; - const isContainer = "items" in payload; - if (isContainer && (payload.version !== 2 || !Array.isArray(payload.items))) return []; - - const events: SentrySpanEvent[] = []; - for (const raw of isContainer ? (payload.items as unknown[]) : [payload]) { - if (!isRecord(raw)) continue; - const end = isContainer ? raw.end_timestamp : raw.timestamp; - if ( - typeof raw.trace_id !== "string" || - !raw.trace_id || - typeof raw.span_id !== "string" || - !raw.span_id || - typeof raw.start_timestamp !== "number" || - !Number.isFinite(raw.start_timestamp) || - typeof end !== "number" || - !Number.isFinite(end) - ) - continue; - - const data: Record = {}; - if (isContainer && isRecord(raw.attributes)) { - for (const [key, attribute] of Object.entries(raw.attributes)) { - if (isRecord(attribute) && "value" in attribute) data[key] = attribute.value; - } - } else if (isRecord(raw.data)) { - Object.assign(data, raw.data); - } - - const op = isContainer ? data["sentry.op"] : raw.op; - const name = isContainer ? raw.name : raw.description; - const span: Span = { - trace_id: raw.trace_id, - span_id: raw.span_id, - parent_span_id: typeof raw.parent_span_id === "string" ? raw.parent_span_id : undefined, - description: typeof name === "string" ? name : undefined, - op: typeof op === "string" ? op : undefined, - start_timestamp: raw.start_timestamp, - timestamp: end, - status: typeof raw.status === "string" ? raw.status : undefined, - data, - }; - - events.push({ - type: "span", - event_id: `${raw.trace_id}:${raw.span_id}`, - timestamp: end, - start_timestamp: raw.start_timestamp, - contexts: { - trace: { - trace_id: raw.trace_id, - span_id: raw.span_id, - parent_span_id: span.parent_span_id ?? undefined, - op: span.op ?? undefined, - status: span.status, - data, - }, - }, - sdk, - span, - is_segment: raw.is_segment === true, - }); - } - return events; -} - -export function processSpanEvent(event: SentrySpanEvent, context: TraceProcessingContext): TraceProcessingResult { - const trace = context.existingTrace ?? initializeTrace(event); - const span = { - ...event.span, - start_timestamp: toTimestamp(event.span.start_timestamp), - timestamp: toTimestamp(event.span.timestamp), - }; - - // A segment serves the same role as the root transaction in the existing UI. - // Child spans must not inflate the transaction count or replace its name. - if (event.is_segment) { - const transaction: SentryTransactionEvent = { - ...event, - type: "transaction", - transaction: span.description ?? "(unknown transaction)", - start_timestamp: event.start_timestamp as SentryTransactionEvent["start_timestamp"], - contexts: { ...event.contexts, trace: event.contexts!.trace! }, - }; - return processTransactionEvent(transaction, { ...context, existingTrace: trace }); - } - - trace.spans.set(span.span_id, span); - rebuildSpanTree(trace); - return { trace, shouldUpdateTrace: true }; -} +export * from "../../../../telemetry/store/utils/spanProcessor.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/utils/traceInitializer.ts b/packages/spotlight/src/ui/telemetry/store/utils/traceInitializer.ts index 147a41999..0011259b8 100644 --- a/packages/spotlight/src/ui/telemetry/store/utils/traceInitializer.ts +++ b/packages/spotlight/src/ui/telemetry/store/utils/traceInitializer.ts @@ -1,29 +1 @@ -import type { SentryEvent, SentryTransactionEvent, Span, Trace } from "../../types"; - -/** - * Creates a new trace object from an event's trace context. - * This is used when we receive an event with a trace_id but don't have a trace yet. - */ -export function initializeTrace(event: SentryEvent): Trace { - // Add guard to ensure contexts, trace context and trace_id exist - if (!event.contexts?.trace || !event.contexts.trace.trace_id) { - throw new Error("Event missing required trace context or trace_id"); - } - - const traceCtx = event.contexts.trace; - - return { - ...traceCtx, - trace_id: traceCtx.trace_id!, - spans: new Map(), - spanTree: [] as Span[], - transactions: [] as SentryTransactionEvent[], - errors: 0, - start_timestamp: event.start_timestamp ?? event.timestamp, - timestamp: event.timestamp, - status: traceCtx.status, - rootTransactionName: event.transaction || "(unknown transaction)", - rootTransaction: null, - profileGrafted: false, - }; -} +export * from "../../../../telemetry/store/utils/traceInitializer.ts"; diff --git a/packages/spotlight/src/ui/telemetry/store/utils/traceProcessor.ts b/packages/spotlight/src/ui/telemetry/store/utils/traceProcessor.ts index 38612c264..9f47b7e93 100644 --- a/packages/spotlight/src/ui/telemetry/store/utils/traceProcessor.ts +++ b/packages/spotlight/src/ui/telemetry/store/utils/traceProcessor.ts @@ -1,163 +1 @@ -import { graftProfileSpans } from "../../data/profiles"; -import type { SentryTransactionEvent, Span, Trace } from "../../types"; -import { compareSpans, groupSpans } from "../../utils/traces"; -import type { SentryProfileWithTraceMeta } from "../types"; -import { toTimestamp } from "../utils"; -import type { ProcessedProfileChunk } from "./profileChunkProcessor"; -import { mergeChunksToProfile } from "./profileChunkProcessor"; - -export interface TraceProcessingContext { - existingTrace?: Trace; - profilesByTraceId: Map; - profileChunksByProfilerId?: Map; -} - -export interface TraceProcessingResult { - trace: Trace; - shouldUpdateTrace: boolean; - /** V2 profile merged from chunks, to be stored in profilesByTraceId */ - mergedProfile?: SentryProfileWithTraceMeta; -} - -/** - * Processes a transaction event and updates the trace data structure. - * Handles span tree construction, profile grafting, and trace metadata. - */ -export function processTransactionEvent( - event: SentryTransactionEvent, - context: TraceProcessingContext, -): TraceProcessingResult { - const traceCtx = event.contexts.trace; - const { existingTrace, profilesByTraceId, profileChunksByProfilerId } = context; - - // Add guard to ensure trace_id exists - if (!traceCtx.trace_id) { - throw new Error("Transaction event missing required trace_id"); - } - - // Initialize or get existing trace - const trace: Trace = existingTrace ?? { - ...traceCtx, - trace_id: traceCtx.trace_id, - spans: new Map(), - spanTree: [] as Span[], - transactions: [] as SentryTransactionEvent[], - errors: 0, - start_timestamp: event.start_timestamp ?? event.timestamp, - timestamp: event.timestamp, - status: traceCtx.status, - rootTransactionName: event.transaction || "(unknown transaction)", - rootTransaction: null, - profileGrafted: false, - }; - - // Update trace timestamps - trace.start_timestamp = Math.min(event.start_timestamp ?? event.timestamp, trace.start_timestamp); - trace.timestamp = Math.max(event.timestamp, trace.timestamp); - - // Add transaction and rebuild span tree - trace.transactions.push(event); - trace.transactions.sort(compareSpans); - - // Keep standalone spans received before this transaction/segment finished. - const spanMap: Map = new Map(trace.spans); - for (const txn of trace.transactions) { - const txnTrace = txn.contexts.trace; - if (!txnTrace || !txnTrace.span_id || !txnTrace.trace_id) { - continue; - } - - spanMap.set(txnTrace.span_id, { - ...txnTrace, - span_id: txnTrace.span_id, - trace_id: txnTrace.trace_id, - tags: txn?.tags, - start_timestamp: txn.start_timestamp, - timestamp: txn.timestamp, - description: traceCtx.description || txn.transaction, - transaction: txn, - }); - - // Add child spans - if (txn.spans) { - for (const span of txn.spans) { - spanMap.set(span.span_id, { - ...span, - timestamp: toTimestamp(span.timestamp), - start_timestamp: toTimestamp(span.start_timestamp), - }); - } - } - } - - trace.spans = spanMap; - rebuildSpanTree(trace); - - // Graft profile data if available (V1 profiles keyed by trace_id) - const profileForTrace = profilesByTraceId.get(trace.trace_id); - if (profileForTrace) { - graftProfileSpans(trace, profileForTrace); - } - - // Try V2 continuous profiling: look for profiler_id in transaction context - // V2 uses contexts.profile.profiler_id to link profiles - let mergedProfile: SentryProfileWithTraceMeta | undefined; - if (!trace.profileGrafted && profileChunksByProfilerId) { - const profileContext = event.contexts?.profile as { profiler_id?: string } | undefined; - const profilerId = profileContext?.profiler_id; - - if (profilerId) { - const chunks = profileChunksByProfilerId.get(profilerId); - if (chunks && chunks.length > 0) { - // Get the active thread ID from trace context data if available - const activeThreadId = traceCtx.data?.["thread.id"] as string | undefined; - mergedProfile = mergeChunksToProfile(chunks, activeThreadId) ?? undefined; - if (mergedProfile) { - graftProfileSpans(trace, mergedProfile); - } - } - } - } - - return { - trace, - shouldUpdateTrace: true, - mergedProfile, - }; -} - -/** Rebuild from real spans so late parents replace placeholders and children are not appended twice. */ -export function rebuildSpanTree(trace: Trace): void { - trace.spans = new Map( - Array.from(trace.spans) - .filter(([, span]) => span.op !== "orphan") - .map(([id, span]) => [id, { ...span, children: undefined }]), - ); - trace.spanTree = groupSpans(trace.spans); -} - -/** - * Updates trace metadata based on its transactions. - * Determines root transaction and handles orphan traces. - */ -export function updateTraceMetadata(trace: Trace): void { - const roots = trace.transactions.filter(e => !e.contexts.trace.parent_span_id); - - if (roots.length === 1) { - trace.rootTransaction = roots[0]; - trace.rootTransactionName = roots[0].transaction || "(unknown transaction)"; - } else if (roots.length > 1) { - trace.rootTransactionName = "(multiple root transactions)"; - } else if (trace.transactions.length > 0) { - // Orphan trace: no root transaction, but has child transactions - console.debug( - `[Spotlight] Orphan trace detected (trace_id: ${trace.trace_id}). ` + - `Using first transaction "${trace.transactions[0].transaction}" as fallback.`, - ); - // use the first transcation for orphan traces - trace.rootTransaction = trace.transactions[0]; - trace.rootTransactionName = trace.transactions[0].transaction || "(orphan transaction)"; - } else { - trace.rootTransactionName = "(missing root transaction)"; - } -} +export * from "../../../../telemetry/store/utils/traceProcessor.ts"; diff --git a/packages/spotlight/src/ui/telemetry/types.ts b/packages/spotlight/src/ui/telemetry/types.ts index 3f6669fdb..fb445eb7d 100644 --- a/packages/spotlight/src/ui/telemetry/types.ts +++ b/packages/spotlight/src/ui/telemetry/types.ts @@ -1,480 +1,7 @@ -import type { EnvelopeItem, EventEnvelopeHeaders, Measurements, SerializedLog } from "@sentry/core"; import type { ColorValue } from "nanovis"; - -export type TraceId = string; -export type SpanId = string; - -export type EventAttachment = { - header: EnvelopeItem[0]; - data: string; -}; - -export type FrameVars = { - [key: string]: string; -}; - -export type EventFrame = { - filename?: string; - abs_path?: string; - function?: string; - module?: string; - lineno?: number; - colno?: number; - pre_context?: string[]; - post_context?: string[]; - context_line?: string; - vars?: FrameVars; - instruction_addr?: string; - in_app?: boolean; -}; - -export type EventStacktrace = { - frames: EventFrame[]; -}; - -export type EventExceptionValue = { - type: string; - value: string; - stacktrace?: EventStacktrace; -}; - -export type EventException = - | { - values: EventExceptionValue[]; - value: undefined; - } - | { - values: undefined; - value: EventExceptionValue; - }; - -export type Breadcrumb = { - data?: Record; - message?: string; - category: string; - timestamp: string; - type: string | "default"; -}; - -export type Context = Record; - -type CommonEventAttrs = { - // not always present, but we are forcing it in EventCache - event_id: string; - timestamp: number; - message?: SentryFormattedMessage; - breadcrumbs?: Breadcrumb[] | { values: Breadcrumb[] }; - transaction?: string; - environment?: string; - platform?: string; - server_name?: string; - release?: string; - start_timestamp?: number; - contexts?: Contexts; - tags?: Tags; - extra?: Context; - request?: Record | string>; - modules?: Record; - sdk?: Sdk; - measurements?: Measurements; - attachments?: EventAttachment[]; - // Inferred source type for distinguishing browser/server/mobile events - __sourceType?: "browser" | "server" | "mobile"; -}; - -// Note: For some reason the `sentry/core` module doesn't have these additional properties -// in `EventEnvelopeHeaders['trace']` but they are present in the actual events. -// Follow up? -export type TraceContext = EventEnvelopeHeaders["trace"] & { - span_id?: string; - status?: "ok" | string; - description?: string; - parent_span_id?: string; - data?: Record; - op?: string; -}; - -export type FeedbackContext = { - message: string; - name?: string; - contact_email?: string; - source?: string; - url?: string; - associated_event_id?: string; - replay_id?: string; -}; - -export type Contexts = { - trace?: TraceContext; - feedback?: FeedbackContext; -} & { - [key: string]: Context | TraceContext | FeedbackContext | undefined; -}; - -export type Tags = { - [key: string]: string; -}; - -export type SentryFormattedMessage = - | string - | { - formatted: string; - params?: []; - }; - -export type SentryErrorEvent = CommonEventAttrs & { - type?: "error" | "event" | "message" | "default"; - level?: "fatal" | "error" | "warning" | "info" | "debug"; - exception: EventException; -}; - -export type Span = { - trace_id?: TraceId; - span_id: SpanId; - parent_span_id?: string | null; - op?: string | null; - description?: string | null; - start_timestamp: number; - tags?: Tags | null; - timestamp: number; - status?: "ok" | string; - transaction?: SentryTransactionEvent; - children?: Span[]; - data?: Record; -}; - -export type SentryTransactionEvent = CommonEventAttrs & { - type: "transaction"; - spans?: Span[]; - start_timestamp: string; - contexts: Contexts & { - trace: TraceContext; - }; -}; - -// Standalone spans are normalized at the envelope boundary. Only segment spans -// are represented as transactions; child spans remain individual spans. -export type SentrySpanEvent = CommonEventAttrs & { - type: "span"; - span: Span; - is_segment: boolean; -}; - -export type ProfileSample = { - elapsed_since_start_ns: string; - stack_id: number; - thread_id: string; -}; - -export type ProcessedProfileSample = { - start_timestamp: number; - stack_id: number; - thread_id: string; -}; - -export type SentryProfile = { - samples: ProfileSample[]; - stacks: number[][]; - frames: EventFrame[]; - platform?: string; - thread_metadata?: Record< - string, - { - name?: string; - priority?: number; - } - >; -}; - -export type SentryProcessedProfile = SentryProfile & { - samples: ProcessedProfileSample[]; -}; - -export type AggregateCallData = { - name: string; - totalTime: number; - samples: number; - traceIds: Set; -}; - -export type SentryDeviceInfo = { - architecture: string; - is_emulator?: boolean; - locale?: string; - manufacturer?: string; - model?: string; -}; - -export type SentryOSInfo = { - name: string; - version: string; - build_number?: string; -}; - -export type SentryProfileTransactionInfo = { - name: string; - id: string; - trace_id: string; - active_thread_id: string; - relative_start_ns?: string; - relative_end_ns?: string; -}; - -export type SentryProfileV1Event = CommonEventAttrs & { - type: "profile"; - device: SentryDeviceInfo; - os: SentryOSInfo; - transactions?: Array; - transaction?: SentryProfileTransactionInfo; - version: "1"; - profile: SentryProfile; -}; - -// V2 Profile (Continuous Profiling) Types -export type ProfileV2Sample = { - timestamp: number; // Unix timestamp in seconds with microseconds precision - stack_id: number; - thread_id: string; -}; - -export type ProfileV2MeasurementValue = { - timestamp: number; - value: number; -}; - -export type ProfileV2Measurement = { - unit: string; - values: ProfileV2MeasurementValue[]; -}; - -export type SentryProfileV2 = { - samples: ProfileV2Sample[]; - stacks: number[][]; - frames: EventFrame[]; - thread_metadata: Record< - string, - { - name?: string; - priority?: number; - } - >; -}; - -export type SentryProfileV2ChunkEvent = { - type: "profile_chunk"; - version: "2"; - profiler_id: string; // Links chunks from same profiler session - chunk_id: string; // Unique ID for this chunk - platform: string; - release?: string; - environment?: string; - client_sdk?: { - name: string; - version: string; - }; - debug_meta?: { - images?: Array<{ - debug_id?: string; - image_addr?: string; - type?: string; - image_size?: number; - code_file?: string; - }>; - }; - measurements?: Record; - profile: SentryProfileV2; -}; - -export type SentryLogEventItem = SerializedLog & { - id: string; // Need to have a unique id for each log - severity_number: number; - sdk: string | undefined; -}; - -export type SentryLogEvent = CommonEventAttrs & { - type: "log"; - items: Array; -}; - -export type SentryFeedbackEvent = CommonEventAttrs & { - type: "feedback"; - contexts: Contexts & { - feedback: FeedbackContext; - }; -}; - -export type SentryEvent = - | SentryErrorEvent - | SentryTransactionEvent - | SentrySpanEvent - | SentryProfileV1Event - | SentryProfileV2ChunkEvent - | SentryLogEvent - | SentryFeedbackEvent; - -export type Trace = TraceContext & { - trace_id: string; - transactions: SentryTransactionEvent[]; - errors: number; - start_timestamp: number; - timestamp: number; - status?: "ok" | string; - rootTransaction: SentryTransactionEvent | null; - rootTransactionName: string; - spans: Map; - spanTree: Span[]; - profileGrafted: boolean; -}; - -export type Sdk = { - name: string; - version: string; - lastSeen: number; -}; - -export type SentryEventWithPerformanceData = Omit & { - measurements: Record< - string, - { - value: number; - unit: string; - } - > & { - "score.total": { - value: number; - unit: string; - }; - "score.fcp": { - value: number; - unit: string; - }; - "score.lcp": { - value: number; - unit: string; - }; - "score.fid": { - value: number; - unit: string; - }; - "score.cls": { - value: number; - unit: string; - }; - "score.ttfb": { - value: number; - unit: string; - }; - "score.weight.fcp": { - value: number; - unit: string; - }; - "score.weight.lcp": { - value: number; - unit: string; - }; - "score.weight.fid": { - value: number; - unit: string; - }; - "score.weight.cls": { - value: number; - unit: string; - }; - "score.weight.ttfb": { - value: number; - unit: string; - }; - }; -}; - -export type MetricScoreProps = { - fcpScore: number; - lcpScore: number; - clsScore: number; - fidScore: number; - ttfbScore: number; -}; -export type MetricWeightsProps = { - fcp: number; - lcp: number; - cls: number; - fid: number; - ttfb: number; -}; - -export type AIToolCall = { - toolCallId: string; - toolName: string; - args: Record; - result?: Record | string; - state?: string; - step?: number; -}; - -// Content can be string (v1) or array of content blocks (v2) -export type AIContentBlock = { - type: string; - text?: string; -}; - -export type AIMessageContent = string | AIContentBlock[]; - -export type AIMessage = { - role: string; - content: AIMessageContent; - toolInvocations?: AIToolCall[]; - parts?: unknown[]; -}; - -export type AIPrompt = { - system?: string; - messages?: AIMessage[]; -}; - -export type AIResponse = { - finishReason?: string; - text?: string; - toolCalls?: AIToolCall[]; -}; - -export type AIMetadata = { - modelId?: string; - modelProvider?: string; - functionId?: string; - metadata: Record; - maxRetries?: number; - maxSteps?: number; - promptTokens?: number; - completionTokens?: number; -}; - -export type SpotlightAITrace = { - id: string; // root span_id, also used as traceId for display - name: string; // root span description or op for primary display - operation: string; - timestamp: number; - durationMs: number; - tokensDisplay: string; // prompt/completion or N/A - promptTokens?: number; - completionTokens?: number; - hasToolCall: boolean; - rawSpan: Span; - metadata: AIMetadata; - prompt?: AIPrompt; - response?: AIResponse; - toolCalls: AIToolCall[]; -}; - -export type AILibraryHandler = { - id: string; - name: string; - canHandleSpan: (span: Span) => boolean; - extractRootSpans: (spans: Span[]) => Span[]; - processTrace: (rootSpan: Span) => SpotlightAITrace; - getDisplayTitle: (trace: SpotlightAITrace) => string; - getTypeBadge: (trace: SpotlightAITrace) => string; - getTokensDisplay: (trace: SpotlightAITrace) => string; -}; +import type { EventFrame } from "../../telemetry/types.ts"; +export type * from "../../telemetry/types.ts"; +export type { AITrace as SpotlightAITrace } from "../../telemetry/types.ts"; /** * A generic tree node used to represent hierarchical performance data diff --git a/packages/spotlight/src/ui/telemetry/utils.ts b/packages/spotlight/src/ui/telemetry/utils.ts index 1a09574cd..5b725f8a1 100644 --- a/packages/spotlight/src/ui/telemetry/utils.ts +++ b/packages/spotlight/src/ui/telemetry/utils.ts @@ -1,10 +1,9 @@ import { removeURLSuffix } from "@spotlight/ui/lib/removeURLSuffix"; import { log } from "../lib/logger"; -import useSentryStore from "./store"; +import { setTelemetrySidecarUrl } from "./store/store"; export function setSidecarUrlInStore(url: string) { - const store = useSentryStore.getState(); const baseSidecarUrl = removeURLSuffix(url, "/stream"); - store.setSidecarUrl(baseSidecarUrl); + setTelemetrySidecarUrl(baseSidecarUrl); log("Set sidecar URL for telemetry:", baseSidecarUrl); } diff --git a/packages/spotlight/src/ui/telemetry/utils/profileUtils.ts b/packages/spotlight/src/ui/telemetry/utils/profileUtils.ts index b51063daf..4b664606b 100644 --- a/packages/spotlight/src/ui/telemetry/utils/profileUtils.ts +++ b/packages/spotlight/src/ui/telemetry/utils/profileUtils.ts @@ -1,15 +1 @@ -import type { EventFrame } from "../types"; - -const _FUNCTION_NAME_FROM_FRAME_CACHE = new Map(); -export function getFunctionNameFromFrame(frame: EventFrame): string { - let result = _FUNCTION_NAME_FROM_FRAME_CACHE.get(frame); - if (!result) { - const module = frame.module || frame.filename || frame.abs_path || ""; - const functionName = frame.function || ""; - const lineNo = frame.lineno ? `:${frame.lineno}` : ""; - const colNo = frame.lineno && frame.colno ? `:${frame.colno}` : ""; - result = `${module}@${functionName}${lineNo}${colNo}`; - _FUNCTION_NAME_FROM_FRAME_CACHE.set(frame, result); - } - return result; -} +export * from "../../../telemetry/utils/profileUtils.ts"; diff --git a/packages/spotlight/src/ui/telemetry/utils/sdkToPlatform.ts b/packages/spotlight/src/ui/telemetry/utils/sdkToPlatform.ts index e7cd4215f..2240482d0 100644 --- a/packages/spotlight/src/ui/telemetry/utils/sdkToPlatform.ts +++ b/packages/spotlight/src/ui/telemetry/utils/sdkToPlatform.ts @@ -1,15 +1 @@ -/** - * Normalizes SDK name to a platform. Can be as specific or unspecific as we support different platforms and SDKs. - */ -export function sdkToPlatform(name: string) { - if (name.includes("javascript.astro")) return "javascript.astro"; - if (name.includes("javascript")) return "javascript"; - if (name.includes("java")) return "java"; - if (name.includes("python")) return "python"; - if (name.includes("php.laravel")) return "php.laravel"; - if (name.includes("php.symfony")) return "php.symfony"; - if (name.includes("php")) return "php"; - if (name.includes("ruby")) return "ruby"; - if (name.includes("dotnet")) return "dotnet"; - return "unknown"; -} +export * from "../../../telemetry/utils/sdkToPlatform.ts"; diff --git a/packages/spotlight/src/ui/telemetry/utils/sentry.ts b/packages/spotlight/src/ui/telemetry/utils/sentry.ts index 2e03bd376..fb25140c7 100644 --- a/packages/spotlight/src/ui/telemetry/utils/sentry.ts +++ b/packages/spotlight/src/ui/telemetry/utils/sentry.ts @@ -1,44 +1 @@ -import { - ERROR_EVENT_TYPES, - FEEDBACK_EVENT_TYPES, - LOG_EVENT_TYPES, - PROFILE_EVENT_TYPES, - TRACE_EVENT_TYPES, -} from "../constants/sentry"; -import type { - SentryErrorEvent, - SentryEvent, - SentryFeedbackEvent, - SentryLogEvent, - SentryProfileV1Event, - SentryProfileV2ChunkEvent, - SentryTransactionEvent, -} from "../types"; - -export function isErrorEvent(event: SentryEvent): event is SentryErrorEvent { - return (!event.type || ERROR_EVENT_TYPES.has(event.type)) && Boolean((event as SentryErrorEvent).exception); -} - -export function isProfileEvent(event: SentryEvent): event is SentryProfileV1Event { - return !!event.type && PROFILE_EVENT_TYPES.has(event.type) && (event as SentryProfileV1Event).version === "1"; -} - -export function isProfileChunkEvent(event: SentryEvent): event is SentryProfileV2ChunkEvent { - return event.type === "profile_chunk" && (event as SentryProfileV2ChunkEvent).version === "2"; -} - -export function isTraceEvent(event: SentryEvent): event is SentryTransactionEvent { - return !!event.type && TRACE_EVENT_TYPES.has(event.type); -} - -export function isLogEvent(event: SentryEvent): event is SentryLogEvent { - return !!event.type && LOG_EVENT_TYPES.has(event.type); -} - -export function isFeedbackEvent(event: SentryEvent): event is SentryFeedbackEvent { - return ( - !!event.type && - FEEDBACK_EVENT_TYPES.has(event.type) && - Boolean((event as SentryFeedbackEvent).contexts?.feedback) - ); -} +export * from "../../../telemetry/utils/sentry.ts"; diff --git a/packages/spotlight/src/ui/telemetry/utils/traces.ts b/packages/spotlight/src/ui/telemetry/utils/traces.ts index 70dc686f6..36c8719c7 100644 --- a/packages/spotlight/src/ui/telemetry/utils/traces.ts +++ b/packages/spotlight/src/ui/telemetry/utils/traces.ts @@ -1,87 +1 @@ -import { log } from "@spotlight/ui/lib/logger"; -import type { Span, Trace } from "../types"; - -// mutates spans in place and adds children, as well as returns the top level tree -export function groupSpans(spans: Map): Span[] { - const tree: Span[] = []; - - // need to sort root(s) first - const sortedSpans = Array.from(spans.values()).sort((a, b) => { - const parentComp = (a.parent_span_id ? 1 : 0) - (b.parent_span_id ? 1 : 0); - return parentComp === 0 ? compareSpans(a, b) : parentComp; - }); - - for (const span of sortedSpans) { - let parent = span && getParentOfSpan(span, spans, sortedSpans); - - span.children ??= []; - if (parent) { - parent.children ??= []; - parent.children.push(span); - } else if (span.parent_span_id) { - const parentParent = sortedSpans.find(s => !s.parent_span_id); - if (!parentParent) { - log(`Root span (${span.parent_span_id}) for span (${span.span_id}). Creating orphan.`); - } else { - log(`Creating orphan for parent (${span.parent_span_id}) for span (${span.span_id})`); - } - parent = { - trace_id: span.trace_id, - span_id: span.parent_span_id, - parent_span_id: parentParent ? parentParent.span_id : null, - op: "orphan", - description: "missing or unknown parent span", - children: [span], - start_timestamp: span.start_timestamp, - timestamp: span.timestamp, - status: "unknown", - }; - spans.set(parent.span_id, parent); - // sortedSpans.splice(spanIdx, 0, parent); - if (parentParent) { - parentParent.children ??= []; - parentParent.children.push(parent); - } else { - tree.push(parent); - } - } else { - tree.push(span); - } - spans.set(span.span_id, span); - } - - return tree; -} - -function getParentOfSpan(span: Span, idLookup: Map, allSpans: Span[]): Span | undefined { - if (!span.parent_span_id) { - return undefined; - } - - return idLookup.get(span.parent_span_id) || allSpans.find(s => s.span_id === span.parent_span_id); -} - -export function compareSpans(a: { start_timestamp: number }, b: { start_timestamp: number }): number { - return a.start_timestamp - b.start_timestamp; -} - -export function getRootTransactionMethod(trace: Trace) { - // Fallback to first transaction if no root transaction (e.g., orphan traces) - const transaction = trace.rootTransaction || trace.transactions[0]; - const method = String(transaction?.contexts?.trace.data?.method || transaction?.request?.method || ""); - return method; -} - -export function getRootTransactionName(trace: Trace) { - const method = getRootTransactionMethod(trace); - // For orphan traces without a root, use the first transaction's name - let name = trace.rootTransactionName; - if (!trace.rootTransaction && trace.transactions.length > 0) { - name = trace.transactions[0].transaction || trace.rootTransactionName; - } - - // Strip method prefix if present - const finalName = method && name.startsWith(method) ? name.slice(method.length + 1) : name; - - return finalName; -} +export * from "../../../telemetry/utils/traces.ts"; diff --git a/packages/spotlight/src/ui/telemetry/utils/webVitals.ts b/packages/spotlight/src/ui/telemetry/utils/webVitals.ts index 6846511ed..921930b29 100644 --- a/packages/spotlight/src/ui/telemetry/utils/webVitals.ts +++ b/packages/spotlight/src/ui/telemetry/utils/webVitals.ts @@ -1,171 +1,5 @@ -import type { SentryEvent } from "@spotlight/ui/telemetry/types"; import type { WebVitals } from "../constants"; - -const SQRT_2 = Math.sqrt(2); - -// Gauss error function -function erf(x: number): number { - const a1 = 0.254829592; - const a2 = -0.284496736; - const a3 = 1.421413741; - const a4 = -1.453152027; - const a5 = 1.061405429; - const p = 0.3275911; - - const sign = x < 0 ? -1 : 1; - const absX = Math.abs(x); - - const t = 1.0 / (1.0 + p * absX); - const y = 1.0 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX); - - return sign * y; -} - -// Sigma function for CDF score calculation -export function calculateCdfSigma(p10: number, p50: number): number { - return Math.abs(Math.log(p10) - Math.log(p50)) / (SQRT_2 * 0.9061938024368232); -} - -// Calculates a log-normal CDF score based on a log-normal with a specific p10 and p50 -export function calculateCdfScore(value: number, p10: number, p50: number): number { - return 0.5 * (1.0 - erf((Math.log(value) - Math.log(p50)) / (SQRT_2 * calculateCdfSigma(p50, p10)))); -} - -interface ScoreComponent { - measurement: string; - weight: number; - p10: number; - p50: number; - optional?: boolean; -} - -interface Profile { - name: string; - scoreComponents: ScoreComponent[]; - condition?: { - op: string; - name: string; - value: string | number; - }; -} - -interface PerformanceScoreConfig { - profiles: Profile[]; -} - -// export function getPropertyByPath(obj: Record, path: string): any { -// const keys = path.split('.'); -// let result = obj; - -// for (const key of keys) { -// if (result && result[key]) { -// result = result[key]; -// } else { -// return undefined; -// } -// } - -// return result; -// } - -// export function matchCondition(condition: { op: string; name: string; value: string | number }, event: SentryEvent): boolean { -// const { op, name, value } = condition; - -// if (op === 'eq') { -// const eventValue = getPropertyByPath({ event }, name); -// return eventValue === value; -// } - -// return false; -// } - -export function normalizePerformanceScore( - event: SentryEvent, - performanceScore: PerformanceScoreConfig | undefined, -): void { - if (!performanceScore) { - return; - } - - for (const profile of performanceScore.profiles) { - // TODO: To check a matching condition as done in relay. Skipped right now because Browser type is not coming in Event data - // const condition = profile.condition; - // if (condition && !matchCondition(condition, event)) { - // continue; - // } - - const measurements = event.measurements; - - if (measurements) { - let shouldAddTotal = false; - - // TODO: To calculate performance score if all required metric fields are present in event measurement data. - // if ( - // profile.scoreComponents.some(c => { - // return ( - // !Object.prototype.hasOwnProperty.call(measurements, c.measurement) && - // Math.abs(c.weight) >= Number.EPSILON && - // !c.optional - // ); - // }) - // ) { - // break; - // } - - let scoreTotal = 0.0; - let weightTotal = 0.0; - - for (const component of profile.scoreComponents) { - if (component.optional && !Object.prototype.hasOwnProperty.call(measurements, component.measurement)) { - continue; - } - - weightTotal += component.weight; - } - - if (Math.abs(weightTotal) < Number.EPSILON) { - break; - } - - for (const component of profile.scoreComponents) { - let normalizedComponentWeight = 0.0; - - if (Object.prototype.hasOwnProperty.call(measurements, component.measurement)) { - normalizedComponentWeight = component.weight / weightTotal; - const value = measurements[component.measurement].value; - const cdf = calculateCdfScore(Math.max(0.0, value), component.p10, component.p50); - const componentScore = cdf * normalizedComponentWeight; - scoreTotal += componentScore; - shouldAddTotal = true; - - measurements[`score.${component.measurement}`] = { - value: cdf, - unit: "ratio", - }; - } else { - measurements[`score.${component.measurement}`] = { - value: 0, - unit: "ratio", - }; - } - - measurements[`score.weight.${component.measurement}`] = { - value: normalizedComponentWeight, - unit: "ratio", - }; - } - - if (shouldAddTotal && !Object.prototype.hasOwnProperty.call(measurements, "score.total")) { - measurements["score.total"] = { - value: scoreTotal, - unit: "ratio", - }; - } - - break; - } - } -} +export { calculateCdfScore, calculateCdfSigma, normalizePerformanceScore } from "../../../telemetry/utils/webVitals.ts"; export function calculateLabelCoordinates( size: number, diff --git a/packages/spotlight/tests/types/sdk-core.ts b/packages/spotlight/tests/types/sdk-core.ts new file mode 100644 index 000000000..f3a08d42c --- /dev/null +++ b/packages/spotlight/tests/types/sdk-core.ts @@ -0,0 +1,29 @@ +import { type ProcessedEvent, type QuerySummary, createTelemetry } from "@spotlightjs/spotlight/sdk/core"; + +const telemetry = createTelemetry(); +const queries: readonly QuerySummary[] = telemetry.getQuerySummaries({ traceId: "trace" }); +const events: readonly ProcessedEvent[] = telemetry.getEvents({ traceId: "trace", spanId: "span" }); +const envelope = telemetry.getEnvelope("envelope"); +if (envelope) { + const [header, items] = envelope; + const sdk = header.sdk; + for (const [itemHeader, payload] of items) { + const type: string = itemHeader.type; + void type; + void payload; + } + void sdk; +} +// @ts-expect-error State storage is private. +telemetry.store; +// @ts-expect-error Public events are read-only. +events.push(events[0]); +// @ts-expect-error Nested maps cannot be mutated through the public type. +telemetry.getTrace("trace")?.spans.clear(); +// @ts-expect-error Duration units are explicit in the public contract. +queries[0].totalTime; +const normalizedTimestamp: number = events[0].timestamp; +void normalizedTimestamp; +void queries[0].totalDurationMs; +void queries[0].averageDurationMs; +telemetry.dispose(); diff --git a/packages/spotlight/tests/types/tsconfig.sdk-core.json b/packages/spotlight/tests/types/tsconfig.sdk-core.json new file mode 100644 index 000000000..f7e6581c4 --- /dev/null +++ b/packages/spotlight/tests/types/tsconfig.sdk-core.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023", "DOM"], + "types": [], + "strict": true, + "noEmit": true, + "skipLibCheck": false + }, + "files": ["sdk-core.ts"] +} diff --git a/packages/spotlight/vite.node.config.ts b/packages/spotlight/vite.node.config.ts index 00d55f659..b7d0ec812 100644 --- a/packages/spotlight/vite.node.config.ts +++ b/packages/spotlight/vite.node.config.ts @@ -43,6 +43,8 @@ export default defineConfig({ run: resolve(__dirname, "src/run.ts"), "server/main": resolve(__dirname, "src/server/main.ts"), "server/sdk": resolve(__dirname, "src/server/sdk.ts"), + "sdk/core": resolve(__dirname, "src/sdk/core.ts"), + "sdk/node": resolve(__dirname, "src/sdk/node.ts"), }, formats: ["es"], }, @@ -71,6 +73,7 @@ export default defineConfig({ "uuidv7", "yaml", "zod", + "zustand/vanilla", ], output: { entryFileNames: "[name].js", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ffb980cd2..a692e9aa4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -175,6 +175,9 @@ importers: zod: specifier: ^4 version: 4.1.13 + zustand: + specifier: ^5.0.3 + version: 5.0.3(@types/react@18.3.20)(react@18.3.1) devDependencies: '@fontsource/raleway': specifier: 'catalog:' @@ -362,9 +365,6 @@ importers: vitest: specifier: 'catalog:' version: 4.1.11(@opentelemetry/api@1.9.0)(@types/node@22.15.28)(@vitest/coverage-v8@4.1.8)(happy-dom@21.0.4)(vite@8.3.4(@types/node@22.15.28)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.3)) - zustand: - specifier: ^5.0.3 - version: 5.0.3(@types/react@18.3.20)(react@18.3.1) packages/website: dependencies: From a0ca9d025d96aff8fd37011fd84bbdb102c6331b Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Tue, 27 Oct 2026 14:56:30 +0530 Subject: [PATCH 2/7] ref(sdk): Distinguish pipelines from ingesters --- packages/spotlight/README.md | 46 ++++++++++--------- packages/spotlight/examples/sdk/frontend.ts | 10 ++-- packages/spotlight/examples/sdk/react.tsx | 8 ++-- packages/spotlight/examples/sdk/server.ts | 12 ++--- packages/spotlight/src/sdk/core.ts | 8 ++-- packages/spotlight/src/sdk/node.spec.ts | 46 +++++++++---------- packages/spotlight/src/sdk/node.ts | 18 ++++---- packages/spotlight/src/server/sdk.spec.ts | 38 +++++++-------- .../src/telemetry/attachments.spec.ts | 8 ++-- .../spotlight/src/telemetry/client.spec.ts | 36 +++++++-------- packages/spotlight/src/telemetry/index.ts | 22 ++++----- .../spotlight/src/telemetry/parity.spec.ts | 12 ++--- packages/spotlight/src/telemetry/public.ts | 10 ++-- .../components/events/EventContexts.test.tsx | 4 +- .../spotlight/src/ui/telemetry/store/store.ts | 10 ++-- packages/spotlight/tests/types/sdk-core.ts | 4 +- 16 files changed, 148 insertions(+), 144 deletions(-) diff --git a/packages/spotlight/README.md b/packages/spotlight/README.md index 33dc45800..1e6e473b8 100644 --- a/packages/spotlight/README.md +++ b/packages/spotlight/README.md @@ -8,26 +8,28 @@ across server and frontend consumers. | Entry | Factory | Input | | --- | --- | --- | -| `@spotlightjs/spotlight/sdk/core` | `createTelemetry()` | Parsed Sentry envelopes; portable across server and frontend | -| `@spotlightjs/spotlight/sdk/node` | `createNodeTelemetry()` | Adds serialized bodies, decompression, and raw payload history | +| `@spotlightjs/spotlight/sdk/core` | `createIngester()` | Parsed Sentry envelopes; portable across server and frontend | +| `@spotlightjs/spotlight/sdk/node` | `createPipeline()` | Adds serialized bodies, decompression, and raw payload history | | `@spotlightjs/spotlight/sdk` | Existing buffer helpers | Compatibility for existing transport consumers | -The core requires no React, DOM, Node globals, or Zustand knowledge. Each instance -owns its state. The SDK does not start a server or synchronize instances; feed +`createIngester()` runs on the frontend or server with parsed envelopes. +`createPipeline()` requires Node APIs for wire decoding and decompression. +Both share all processing and query logic. The core requires no React, DOM, +Node globals, or Zustand knowledge. Each instance owns its state. The SDK does not start a server or synchronize instances; feed server and frontend instances the same envelope stream when both need processing. Match retention settings when comparing their retained histories. ## Server: receive envelopes and read insights ```ts -import { createNodeTelemetry } from "@spotlightjs/spotlight/sdk/node"; +import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; -const telemetry = createNodeTelemetry({ +const pipeline = createPipeline({ rawBufferSize: 100, retention: { maxEnvelopes: 1000, maxAgeMs: 10 * 60 * 1000 }, }); -const result = await telemetry.ingest({ +const result = await pipeline.ingest({ body: Buffer.from(envelopeBody), // Defaults to application/x-sentry-envelope. // contentType: request.headers["content-type"], @@ -38,9 +40,9 @@ const result = await telemetry.ingest({ if (result.status === "invalid") { console.error(result.diagnostics); } else if (result.status === "processed") { - const trace = telemetry.getTrace(traceId); - const queries = telemetry.getQuerySummaries({ traceId }); - const raw = telemetry.getRawPayload(result.envelopeId); // defensive byte copy + const trace = pipeline.getTrace(traceId); + const queries = pipeline.getQuerySummaries({ traceId }); + const raw = pipeline.getRawPayload(result.envelopeId); // defensive byte copy } ``` @@ -51,17 +53,17 @@ Call `server.listen(...)` to start that example; closing it disposes its SDK ins ## Frontend: process the same stream ```ts -import { createTelemetry } from "@spotlightjs/spotlight/sdk/core"; +import { createIngester } from "@spotlightjs/spotlight/sdk/core"; -const telemetry = createTelemetry(); +const ingester = createIngester(); // Your SSE/WebSocket handler supplies parsed envelopes. -await telemetry.ingestEnvelope(parsedEnvelope); +await ingester.ingestEnvelope(parsedEnvelope); -const queries = telemetry.getQuerySummaries({ traceId }); +const queries = ingester.getQuerySummaries({ traceId }); // [{ description, count, totalDurationMs, averageDurationMs, spans }] -const logs = telemetry.getLogs({ traceId }); -const spans = telemetry.getSpans({ traceId }); +const logs = ingester.getLogs({ traceId }); +const spans = ingester.getSpans({ traceId }); ``` See [the frontend lifecycle example](examples/sdk/frontend.ts). Create an instance @@ -71,13 +73,13 @@ once for the view or application, then call `dispose()` when it is no longer nee ```tsx import { useSyncExternalStore } from "react"; -import type { Telemetry } from "@spotlightjs/spotlight/sdk/core"; +import type { Ingester } from "@spotlightjs/spotlight/sdk/core"; -function TraceCount({ telemetry }: { telemetry: Telemetry }) { +function TraceCount({ ingester }: { ingester: Ingester }) { const snapshot = useSyncExternalStore( - telemetry.subscribe, - telemetry.getSnapshot, - telemetry.getSnapshot, + ingester.subscribe, + ingester.getSnapshot, + ingester.getSnapshot, ); return {snapshot.traces.length} traces; } @@ -140,7 +142,7 @@ at that same update. A subscriber or diagnostic observer cannot break ingestion. - Processed retention defaults to **500 envelopes**, including telemetry derived from them. Set `retention.maxEnvelopes` to another positive integer. - Optional `retention.maxAgeMs` measures age since receipt. Expiration runs while - idle, and `await telemetry.prune()` can enforce limits explicitly. + idle, and `await ingester.prune()` can enforce limits explicitly. - Eviction rebuilds derived state from retained envelopes so traces, logs, profiles, and insights stay consistent. Enrichment is reused; generated IDs remain stable. Rebuilding costs CPU proportional to retained telemetry; use smaller limits for diff --git a/packages/spotlight/examples/sdk/frontend.ts b/packages/spotlight/examples/sdk/frontend.ts index cd4a85c03..7e2662a6c 100644 --- a/packages/spotlight/examples/sdk/frontend.ts +++ b/packages/spotlight/examples/sdk/frontend.ts @@ -1,12 +1,12 @@ -import { createTelemetry } from "@spotlightjs/spotlight/sdk/core"; +import { createIngester } from "@spotlightjs/spotlight/sdk/core"; /** Keep one instance for the lifetime of the telemetry view. */ export function createFrontendTelemetry() { - const telemetry = createTelemetry({ retention: { maxEnvelopes: 500 } }); + const ingester = createIngester({ retention: { maxEnvelopes: 500 } }); return { - telemetry, + ingester, // Pass the parsed envelopes received over SSE, WebSocket, or your own transport. - receiveEnvelope: (envelope: unknown) => telemetry.ingestEnvelope(envelope), - disconnect: telemetry.dispose, + receiveEnvelope: (envelope: unknown) => ingester.ingestEnvelope(envelope), + disconnect: ingester.dispose, }; } diff --git a/packages/spotlight/examples/sdk/react.tsx b/packages/spotlight/examples/sdk/react.tsx index 828f3bf3e..38d672f2b 100644 --- a/packages/spotlight/examples/sdk/react.tsx +++ b/packages/spotlight/examples/sdk/react.tsx @@ -1,9 +1,9 @@ -import type { Telemetry } from "@spotlightjs/spotlight/sdk/core"; +import type { Ingester } from "@spotlightjs/spotlight/sdk/core"; import { useSyncExternalStore } from "react"; -export function QueryList({ telemetry, traceId }: { telemetry: Telemetry; traceId?: string }) { - useSyncExternalStore(telemetry.subscribe, telemetry.getSnapshot, telemetry.getSnapshot); - const queries = telemetry.getQuerySummaries({ traceId }); +export function QueryList({ ingester, traceId }: { ingester: Ingester; traceId?: string }) { + useSyncExternalStore(ingester.subscribe, ingester.getSnapshot, ingester.getSnapshot); + const queries = ingester.getQuerySummaries({ traceId }); return (
    {queries.map(query => ( diff --git a/packages/spotlight/examples/sdk/server.ts b/packages/spotlight/examples/sdk/server.ts index cbec4563d..5e4a8058f 100644 --- a/packages/spotlight/examples/sdk/server.ts +++ b/packages/spotlight/examples/sdk/server.ts @@ -1,9 +1,9 @@ import { createServer } from "node:http"; -import { createNodeTelemetry } from "@spotlightjs/spotlight/sdk/node"; +import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; /** Embed the SDK in an HTTP server; the SDK itself does not open a port. */ export function createTelemetryServer() { - const telemetry = createNodeTelemetry({ + const pipeline = createPipeline({ rawBufferSize: 100, retention: { maxEnvelopes: 1000, maxAgeMs: 10 * 60 * 1000 }, }); @@ -14,7 +14,7 @@ export function createTelemetryServer() { if (request.method === "POST" && url.pathname === "/envelope") { const chunks: Buffer[] = []; for await (const chunk of request) chunks.push(Buffer.from(chunk)); - const result = await telemetry.ingest({ + const result = await pipeline.ingest({ body: Buffer.concat(chunks), contentType: request.headers["content-type"], encoding: request.headers["content-encoding"], @@ -26,7 +26,7 @@ export function createTelemetryServer() { } if (request.method === "GET" && url.pathname === "/queries") { const traceId = url.searchParams.get("traceId") ?? undefined; - response.end(JSON.stringify(telemetry.getQuerySummaries({ traceId }))); + response.end(JSON.stringify(pipeline.getQuerySummaries({ traceId }))); return; } response.statusCode = 404; @@ -39,6 +39,6 @@ export function createTelemetryServer() { } } }); - server.on("close", telemetry.dispose); - return { server, telemetry }; + server.on("close", pipeline.dispose); + return { server, pipeline }; } diff --git a/packages/spotlight/src/sdk/core.ts b/packages/spotlight/src/sdk/core.ts index cd5e53a05..2da2fb3b4 100644 --- a/packages/spotlight/src/sdk/core.ts +++ b/packages/spotlight/src/sdk/core.ts @@ -1,8 +1,8 @@ -export { createTelemetry } from "../telemetry/index.ts"; +export { createIngester } from "../telemetry/index.ts"; export type { - Telemetry, - TelemetryOptions, - TelemetrySnapshot, + Ingester, + IngesterOptions, + IngesterSnapshot, RawEnvelope, ProcessedEvent, ProcessedSpan, diff --git a/packages/spotlight/src/sdk/node.spec.ts b/packages/spotlight/src/sdk/node.spec.ts index e6009b882..961e8bf30 100644 --- a/packages/spotlight/src/sdk/node.spec.ts +++ b/packages/spotlight/src/sdk/node.spec.ts @@ -2,7 +2,7 @@ import { readFileSync } from "node:fs"; import { brotliCompressSync, deflateSync, gzipSync } from "node:zlib"; import { describe, expect, test, vi } from "vitest"; -import { createNodeTelemetry } from "./node.ts"; +import { createPipeline } from "./node.ts"; const fixture = readFileSync(new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url)); @@ -16,7 +16,7 @@ describe("Node Telemetry adapter", () => { Buffer.from(`${JSON.stringify({ type, length: binary.length })}\n`), binary, ]); - const telemetry = createNodeTelemetry(); + const telemetry = createPipeline(); const result = await telemetry.ingest({ body }); expect(result.status).toBe("processed"); expect(telemetry.getEvent("with-opaque-item")).toBeDefined(); @@ -30,7 +30,7 @@ describe("Node Telemetry adapter", () => { test.each(['{}\n{"type":"attachment","length":0}', '{}\n{"type":"attachment"}'])( "rejects a truncated item header without its terminating newline: %s", async body => { - const telemetry = createNodeTelemetry(); + const telemetry = createPipeline(); expect((await telemetry.ingest({ body })).status).toBe("invalid"); expect(telemetry.getEnvelopes()).toEqual([]); telemetry.dispose(); @@ -38,7 +38,7 @@ describe("Node Telemetry adapter", () => { ); test("accepts an empty attachment payload after a complete item header", async () => { - const telemetry = createNodeTelemetry(); + const telemetry = createPipeline(); expect((await telemetry.ingest({ body: '{}\n{"type":"attachment","length":0}\n' })).status).toBe("ignored"); expect(telemetry.getEnvelopes()).toHaveLength(1); telemetry.dispose(); @@ -48,7 +48,7 @@ describe("Node Telemetry adapter", () => { ["processed", fixture], ["unsupported", Buffer.from('{}\n{"type":"attachment","length":5}\nhello')], ] as const)("subscribers observe %s raw bytes together with the envelope in one notification", async (_, body) => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const observations: (Buffer | undefined)[] = []; spotlight.subscribe(snapshot => { const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); @@ -59,7 +59,7 @@ describe("Node Telemetry adapter", () => { }); test("raw history bounds already apply when subscribers observe the next envelope", async () => { - const spotlight = createNodeTelemetry({ rawBufferSize: 1 }); + const spotlight = createPipeline({ rawBufferSize: 1 }); const first = await spotlight.ingest({ body: fixture }); if (first.status !== "processed") throw new Error("Expected processed envelope"); const observations: { old: Buffer | undefined; newest: Buffer | undefined }[] = []; @@ -75,7 +75,7 @@ describe("Node Telemetry adapter", () => { }); test("defaults to Sentry envelopes and exposes processed raw payloads", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const result = await spotlight.ingest({ body: fixture }); expect(result.status).toBe("processed"); if (result.status !== "processed") throw new Error("Expected processed envelope"); @@ -92,7 +92,7 @@ describe("Node Telemetry adapter", () => { ["deflate", deflateSync], ["br", brotliCompressSync], ] as const)("decodes %s payloads", async (encoding, compress) => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const result = await spotlight.ingest({ body: new Uint8Array(compress(fixture)), contentType: " Application/X-Sentry-Envelope ; charset=utf-8 ", @@ -105,12 +105,12 @@ describe("Node Telemetry adapter", () => { }); test("accepts serialized envelope strings", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); expect((await spotlight.ingest({ body: fixture.toString("utf8") })).status).toBe("processed"); }); test("returns an explicit ignored result for unsupported content types", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); expect(await spotlight.ingest({ body: fixture, contentType: "application/json" })).toMatchObject({ status: "ignored", reason: expect.any(String), @@ -119,7 +119,7 @@ describe("Node Telemetry adapter", () => { }); test("retains raw payloads for valid envelopes containing only unsupported items", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const body = Buffer.from('{}\n{"type":"attachment","length":5}\nhello'); const result = await spotlight.ingest({ body }); expect(result).toMatchObject({ status: "ignored", reason: "unsupported_items", envelopeId: expect.any(String) }); @@ -150,7 +150,7 @@ describe("Node Telemetry adapter", () => { const warnings = vi.spyOn(console, "warn").mockImplementation(() => {}); const logs = vi.spyOn(console, "log").mockImplementation(() => {}); try { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); // Runtime validation is part of the public boundary. const result = await spotlight.ingest(input as Parameters[0]); expect(result).toMatchObject({ status: "invalid", diagnostics: [{ code }] }); @@ -164,7 +164,7 @@ describe("Node Telemetry adapter", () => { }); test("copies incoming and returned raw payload bytes", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const body = Buffer.from(fixture); const pending = spotlight.ingest({ body }); body.fill(0); @@ -178,7 +178,7 @@ describe("Node Telemetry adapter", () => { }); test("bounds raw payload history independently from processed retention", async () => { - const spotlight = createNodeTelemetry({ rawBufferSize: 1 }); + const spotlight = createPipeline({ rawBufferSize: 1 }); const first = await spotlight.ingest({ body: fixture }); const second = await spotlight.ingest({ body: fixture }); if (first.status !== "processed" || second.status !== "processed") throw new Error("Expected envelopes"); @@ -191,7 +191,7 @@ describe("Node Telemetry adapter", () => { }); test("invalid ingestion cannot evict a retained raw payload", async () => { - const spotlight = createNodeTelemetry({ rawBufferSize: 1 }); + const spotlight = createPipeline({ rawBufferSize: 1 }); const first = await spotlight.ingest({ body: fixture }); if (first.status !== "processed") throw new Error("Expected processed envelope"); expect((await spotlight.ingest({ body: "not an envelope" })).status).toBe("invalid"); @@ -202,7 +202,7 @@ describe("Node Telemetry adapter", () => { }); test("retains at most 500 raw payloads by default", async () => { - const spotlight = createNodeTelemetry({ retention: { maxEnvelopes: 1 } }); + const spotlight = createPipeline({ retention: { maxEnvelopes: 1 } }); const first = await spotlight.ingest({ body: fixture }); let last = first; for (let i = 0; i < 500; i++) last = await spotlight.ingest({ body: fixture }); @@ -213,7 +213,7 @@ describe("Node Telemetry adapter", () => { }); test("zero raw buffer capacity still processes telemetry", async () => { - const spotlight = createNodeTelemetry({ rawBufferSize: 0 }); + const spotlight = createPipeline({ rawBufferSize: 0 }); const result = await spotlight.ingest({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); @@ -222,7 +222,7 @@ describe("Node Telemetry adapter", () => { test("processed age retention does not expire the separate raw history", async () => { vi.useFakeTimers(); - const spotlight = createNodeTelemetry({ retention: { maxAgeMs: 100 } }); + const spotlight = createPipeline({ retention: { maxAgeMs: 100 } }); try { const result = await spotlight.ingest({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); @@ -244,7 +244,7 @@ describe("Node Telemetry adapter", () => { const gate = new Promise(resolve => { release = resolve; }); - const spotlight = createNodeTelemetry({ + const spotlight = createPipeline({ enrichStacktrace: async () => { start(); await gate; @@ -272,7 +272,7 @@ describe("Node Telemetry adapter", () => { test("delivers invalid-input diagnostics to the configured observer", async () => { const diagnostics: string[] = []; - const spotlight = createNodeTelemetry({ + const spotlight = createPipeline({ onDiagnostic: diagnostic => { diagnostics.push(diagnostic.code); }, @@ -283,7 +283,7 @@ describe("Node Telemetry adapter", () => { }); test("throwing diagnostic observers do not reject invalid input", async () => { - const spotlight = createNodeTelemetry({ + const spotlight = createPipeline({ onDiagnostic: () => { throw new Error("observer failed"); }, @@ -292,7 +292,7 @@ describe("Node Telemetry adapter", () => { }); test("reset from a subscriber cannot repopulate raw payload history", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); let reset = false; spotlight.subscribe(() => { if (!reset && spotlight.getEnvelopes().length > 0) { @@ -307,7 +307,7 @@ describe("Node Telemetry adapter", () => { }); test("dispose clears raw payloads and prevents subsequent ingestion", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const result = await spotlight.ingest({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); spotlight.dispose(); diff --git a/packages/spotlight/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts index 7692ca2d3..65246c5bb 100644 --- a/packages/spotlight/src/sdk/node.ts +++ b/packages/spotlight/src/sdk/node.ts @@ -1,9 +1,11 @@ import { Buffer } from "node:buffer"; import { brotliDecompressSync, gunzipSync, inflateSync } from "node:zlib"; import { processEnvelope } from "../server/parser/processEnvelope.ts"; -import { type Diagnostic, type IngestResult, type Telemetry, type TelemetryOptions, createTelemetry } from "./core.ts"; +import { type Diagnostic, type IngestResult, type Ingester, type IngesterOptions, createIngester } from "./core.ts"; -export type NodeTelemetryOptions = TelemetryOptions & { +export type { IngesterSnapshot as PipelineSnapshot } from "./core.ts"; + +export type PipelineOptions = IngesterOptions & { /** * Maximum decoded raw payloads to retain (default 500); zero disables storage. * This history is independent of processed telemetry retention and prune(). @@ -11,7 +13,7 @@ export type NodeTelemetryOptions = TelemetryOptions & { rawBufferSize?: number; }; -export type NodeIngestInput = { +export type PipelineInput = { body: Uint8Array | string; contentType?: string; encoding?: string; @@ -19,8 +21,8 @@ export type NodeIngestInput = { }; export type RawPayload = { body: Buffer; contentType: string }; -export type NodeTelemetry = Telemetry & { - ingest(input: NodeIngestInput): Promise; +export type Pipeline = Ingester & { + ingest(input: PipelineInput): Promise; getRawPayload(envelopeId: string): RawPayload | undefined; }; @@ -31,13 +33,13 @@ const decompressors = new Map Buffer>([ ["br", brotliDecompressSync], ]); -/** Decode serialized Sentry envelopes into an independent Telemetry processing instance. */ -export function createNodeTelemetry(options: NodeTelemetryOptions = {}): NodeTelemetry { +/** Decode serialized Sentry envelopes with Node APIs, then process them through the shared ingester. */ +export function createPipeline(options: PipelineOptions = {}): Pipeline { const rawBufferSize = options.rawBufferSize ?? 500; if (!Number.isSafeInteger(rawBufferSize) || rawBufferSize < 0) { throw new RangeError("rawBufferSize must be a non-negative safe integer"); } - const telemetry = createTelemetry(options); + const telemetry = createIngester(options); const rawPayloads = new Map(); const stagedPayloads = new Map(); let generation = 0; diff --git a/packages/spotlight/src/server/sdk.spec.ts b/packages/spotlight/src/server/sdk.spec.ts index 0e534af52..66ae75448 100644 --- a/packages/spotlight/src/server/sdk.spec.ts +++ b/packages/spotlight/src/server/sdk.spec.ts @@ -3,7 +3,7 @@ import { gzipSync } from "node:zlib"; // @vitest-environment node import type { Envelope } from "@sentry/core"; import { describe, expect, test } from "vitest"; -import { createNodeTelemetry } from "../sdk/node.ts"; +import { createPipeline } from "../sdk/node.ts"; import * as sdk from "./sdk.ts"; const traceId = "a9eab6940aba4ae7bf2f8ac5a54c6a2f"; @@ -34,7 +34,7 @@ function span(overrides: Record = {}) { describe("shared Telemetry SDK", () => { test("ingests compressed v2 spans into traces and query summaries without a browser", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const result = await spotlight.ingest({ body: gzipSync(fixture), encoding: "gzip", contentType }); if (result.status !== "processed") throw new Error("Expected processed envelope"); expect(spotlight.getQuerySummaries()).toMatchObject([ @@ -77,8 +77,8 @@ describe("shared Telemetry SDK", () => { ], }); const before = JSON.stringify(input); - const first = createNodeTelemetry(); - const second = createNodeTelemetry(); + const first = createPipeline(); + const second = createPipeline(); await first.ingestEnvelope(input); await second.ingestEnvelope(input); await first.ingestEnvelope(input); @@ -88,7 +88,7 @@ describe("shared Telemetry SDK", () => { }); test("keeps child spans when parents arrive later and deduplicates repeated delivery", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); await spotlight.ingestEnvelope(envelope("span", { version: 2, items: [span()] })); await spotlight.ingestEnvelope( envelope( @@ -111,7 +111,7 @@ describe("shared Telemetry SDK", () => { }); test("can scope queries to a trace without merging spans that share an ID", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); await spotlight.ingestEnvelope( envelope("span", { version: 2, items: [span(), span({ trace_id: "other-trace" })] }), ); @@ -123,7 +123,7 @@ describe("shared Telemetry SDK", () => { test.each([true, false])( "merges a legacy transaction and streamed segment for the same span (legacy first: %s)", async legacyFirst => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const legacy = envelope( "transaction", { @@ -176,8 +176,8 @@ describe("shared Telemetry SDK", () => { ); test("isolates instance state and resets buffer and processed data together", async () => { - const first = createNodeTelemetry(); - const second = createNodeTelemetry(); + const first = createPipeline(); + const second = createPipeline(); const ingested = await first.ingest({ body: fixture, contentType }); expect(second.getTraces()).toEqual([]); expect(second.getSdks()).toEqual([]); @@ -189,7 +189,7 @@ describe("shared Telemetry SDK", () => { }); test("awaits optional stacktrace enrichment before exposing errors and trace correlation", async () => { - const spotlight = createNodeTelemetry({ + const spotlight = createPipeline({ enrichStacktrace: async event => { await Promise.resolve(); event.exception!.values![0].stacktrace!.frames[0].context_line = "throw new Error()"; @@ -216,7 +216,7 @@ describe("shared Telemetry SDK", () => { const gate = new Promise(resolve => { release = resolve; }); - const spotlight = createNodeTelemetry({ enrichStacktrace: () => gate }); + const spotlight = createPipeline({ enrichStacktrace: () => gate }); const pending = spotlight.ingestEnvelope( envelope("event", { event_id: "error", @@ -236,7 +236,7 @@ describe("shared Telemetry SDK", () => { test.each(["sdks", "logs", "events"] as const)( "reset from a %s subscriber cancels the current ingestion", async key => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); let reset = false; spotlight.subscribe(state => { if (!reset && state[key].length > 0) { @@ -264,7 +264,7 @@ describe("shared Telemetry SDK", () => { ); test("normalizes logs and correlates them to traces", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); await spotlight.ingestEnvelope( envelope("log", { type: "log", @@ -275,7 +275,7 @@ describe("shared Telemetry SDK", () => { }); test("redelivery of the same idless parsed envelope does not duplicate logs", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const input = [ {}, [[{ type: "log" }, { items: [{ timestamp: 1, body: "hello", level: "info", attributes: {} }] }]], @@ -292,7 +292,7 @@ describe("shared Telemetry SDK", () => { }); test("notifies subscribers with completed state and supports unsubscribing", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const observed: number[] = []; const unsubscribe = spotlight.subscribe(() => observed.push(spotlight.getQuerySummaries().length)); await spotlight.ingest({ body: fixture, contentType }); @@ -314,7 +314,7 @@ describe("shared Telemetry SDK", () => { const gate = new Promise(resolve => { release = resolve; }); - const spotlight = createNodeTelemetry({ enrichStacktrace: () => gate }); + const spotlight = createPipeline({ enrichStacktrace: () => gate }); const input = envelope("event", { event_id: "pending", type: "event", timestamp: 1, exception: { values: [] } }); const first = spotlight.ingestEnvelope(input); let secondFinished = false; @@ -329,7 +329,7 @@ describe("shared Telemetry SDK", () => { }); test("an unavailable enrichment adapter does not discard a captured error", async () => { - const spotlight = createNodeTelemetry({ + const spotlight = createPipeline({ enrichStacktrace: async () => { throw new Error("offline"); }, @@ -341,7 +341,7 @@ describe("shared Telemetry SDK", () => { }); test("shares resource summaries, AI extraction, and web-vitals scoring", async () => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); await spotlight.ingestEnvelope( envelope("span", { version: 2, @@ -395,7 +395,7 @@ describe("shared Telemetry SDK", () => { test.each([true, false])( "associates continuous profiles with their trace (profile first: %s)", async profileFirst => { - const spotlight = createNodeTelemetry(); + const spotlight = createPipeline(); const transaction = readFileSync( new URL("../../_fixtures/continuous_profiling/transaction.txt", import.meta.url), ); diff --git a/packages/spotlight/src/telemetry/attachments.spec.ts b/packages/spotlight/src/telemetry/attachments.spec.ts index 69f285a65..16d2b8ba4 100644 --- a/packages/spotlight/src/telemetry/attachments.spec.ts +++ b/packages/spotlight/src/telemetry/attachments.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { createTelemetry } from "../sdk/core.ts"; +import { createIngester } from "../sdk/core.ts"; describe("parsed protocol attachments", () => { test.each([ @@ -11,7 +11,7 @@ describe("parsed protocol attachments", () => { ] as const)( "associates %s with its event and retains canonical immutable data", async (_, payload, contentType, data) => { - const telemetry = createTelemetry(); + const telemetry = createIngester(); const input = [ {}, [ @@ -31,7 +31,7 @@ describe("parsed protocol attachments", () => { ); test("retains opaque unsupported items without discarding their accompanying event", async () => { - const telemetry = createTelemetry(); + const telemetry = createIngester(); const result = await telemetry.ingestEnvelope([ {}, [ @@ -52,7 +52,7 @@ describe("parsed protocol attachments", () => { }); test("continues to reject primitive payloads for supported telemetry items", async () => { - const telemetry = createTelemetry(); + const telemetry = createIngester(); const result = await telemetry.ingestEnvelope([{}, [[{ type: "event" }, "not a parsed event"]]]); expect(result.status).toBe("invalid"); expect(telemetry.getEnvelopes()).toEqual([]); diff --git a/packages/spotlight/src/telemetry/client.spec.ts b/packages/spotlight/src/telemetry/client.spec.ts index 3a39e23c8..40d4e52f7 100644 --- a/packages/spotlight/src/telemetry/client.spec.ts +++ b/packages/spotlight/src/telemetry/client.spec.ts @@ -26,7 +26,7 @@ function spans(id: string, traceId = "trace", count = 1) { afterEach(() => vi.useRealTimers()); test("canonical reads have consistent filters and explicit summary units", async () => { - const client = core.createTelemetry(); + const client = core.createIngester(); const result = await client.ingestEnvelope(spans("one")); expect(result).toMatchObject({ status: "processed", envelopeId: "one", diagnostics: [] }); expect(client.getQuerySummaries({ traceId: "trace" })).toMatchObject([ @@ -40,7 +40,7 @@ test("canonical reads have consistent filters and explicit summary units", async }); test("subscribers see exactly one coherent update per ingestion", async () => { - const client = core.createTelemetry(); + const client = core.createIngester(); const seen: number[] = []; client.subscribe(snapshot => { expect(snapshot.events).toHaveLength(3); @@ -57,7 +57,7 @@ test("subscribers see exactly one coherent update per ingestion", async () => { }); test("returned events, nested maps and earlier snapshots cannot mutate engine state", async () => { - const client = core.createTelemetry(); + const client = core.createIngester(); await client.ingestEnvelope(spans("one")); const snapshot = client.getSnapshot(); const trace = client.getTrace("trace")!; @@ -75,7 +75,7 @@ test("returned events, nested maps and earlier snapshots cannot mutate engine st }); test("snapshots preserve own prototype-named JSON fields without inheriting their values", async () => { - const client = core.createTelemetry(); + const client = core.createIngester(); const input = JSON.parse( '[{"__spotlight_envelope_id":"proto"},[[{"type":"event"},{"event_id":"proto","timestamp":1,"extra":{"__proto__":{"unexpected":true},"kept":1}}]]]', ); @@ -91,7 +91,7 @@ test("snapshots preserve own prototype-named JSON fields without inheriting thei }); test("invalid and unsupported envelopes return diagnostics without changing the snapshot", async () => { - const client = core.createTelemetry(); + const client = core.createIngester(); const initial = client.getSnapshot(); expect(await client.ingestEnvelope(null)).toMatchObject({ status: "invalid", @@ -111,7 +111,7 @@ test("invalid and unsupported envelopes return diagnostics without changing the test("invalid parsed input reports the same diagnostic to the observer and caller", async () => { const diagnostics: string[] = []; - const client = core.createTelemetry({ onDiagnostic: diagnostic => diagnostics.push(diagnostic.code) }); + const client = core.createIngester({ onDiagnostic: diagnostic => diagnostics.push(diagnostic.code) }); const result = await client.ingestEnvelope(null); expect(diagnostics).toEqual(["invalid_envelope"]); expect(result.diagnostics.map(diagnostic => diagnostic.code)).toEqual(diagnostics); @@ -128,7 +128,7 @@ test("a rejected envelope waits for sibling enrichment and cannot poison later e entered = resolve; }); let enrichments = 0; - const client = core.createTelemetry({ + const client = core.createIngester({ enrichStacktrace: async event => { enrichments++; if (enrichments === 1) { @@ -163,7 +163,7 @@ test("a rejected envelope waits for sibling enrichment and cannot poison later e }); test("malformed spans have diagnostics while mixed valid spans still process", async () => { - const client = core.createTelemetry(); + const client = core.createIngester(); expect(await client.ingestEnvelope([{}, [[{ type: "span" }, { version: 2, items: [{}] }]]])).toMatchObject({ status: "invalid", diagnostics: [{ code: "invalid_span" }], @@ -180,8 +180,8 @@ test("malformed spans have diagnostics while mixed valid spans still process", a test("retention preserves identities of profile spans derived from retained envelopes", async () => { const { readFileSync } = await import("node:fs"); - const { createNodeTelemetry } = await import("../sdk/node.ts"); - const client = createNodeTelemetry({ retention: { maxEnvelopes: 3 } }); + const { createPipeline } = await import("../sdk/node.ts"); + const client = createPipeline({ retention: { maxEnvelopes: 3 } }); await client.ingestEnvelope(spans("evict-me")); for (const file of ["transaction.txt", "profile_chunk_1.txt"]) { const body = readFileSync(new URL(`../../_fixtures/continuous_profiling/${file}`, import.meta.url)); @@ -197,7 +197,7 @@ test("retention preserves identities of profile spans derived from retained enve }); test("count retention evicts related events, traces, logs and profiles coherently", async () => { - const client = core.createTelemetry({ retention: { maxEnvelopes: 1 } }); + const client = core.createIngester({ retention: { maxEnvelopes: 1 } }); await client.ingestEnvelope([ { __spotlight_envelope_id: "old" }, [ @@ -231,7 +231,7 @@ test("count retention evicts related events, traces, logs and profiles coherentl test("retention replay preserves generated log IDs and does not repeat enrichment", async () => { let enrichments = 0; - const client = core.createTelemetry({ + const client = core.createIngester({ retention: { maxEnvelopes: 2 }, enrichStacktrace: async () => { enrichments++; @@ -266,7 +266,7 @@ test("duplicate error IDs have one deterministic enrichment owner across replay" entered = resolve; }); let calls = 0; - const client = core.createTelemetry({ + const client = core.createIngester({ retention: { maxEnvelopes: 2 }, enrichStacktrace: async () => { calls++; @@ -297,7 +297,7 @@ test("duplicate error IDs have one deterministic enrichment owner across replay" test("evicting an error owner enriches the retained duplicate from its own exception", async () => { const seen: string[] = []; - const client = core.createTelemetry({ + const client = core.createIngester({ retention: { maxEnvelopes: 3 }, enrichStacktrace: async event => { const exception = event.exception.values![0]; @@ -323,7 +323,7 @@ test("evicting an error owner enriches the retained duplicate from its own excep test("SDK last-seen times do not advance when retained envelopes replay", async () => { vi.useFakeTimers(); - const client = core.createTelemetry({ retention: { maxEnvelopes: 2 } }); + const client = core.createIngester({ retention: { maxEnvelopes: 2 } }); for (const [index, name] of ["old", "keep", "new"].entries()) { vi.setSystemTime(1000 * (index + 1)); await client.ingestEnvelope([ @@ -337,7 +337,7 @@ test("SDK last-seen times do not advance when retained envelopes replay", async test("age retention expires while idle and disposes its timer", async () => { vi.useFakeTimers(); - const client = core.createTelemetry({ retention: { maxAgeMs: 1000 } }); + const client = core.createIngester({ retention: { maxAgeMs: 1000 } }); await client.ingestEnvelope(spans("old")); await vi.advanceTimersByTimeAsync(1000); expect(client.getTraces()).toEqual([]); @@ -358,7 +358,7 @@ test("reset invalidates active and queued work while allowing immediate new inge const ready = new Promise(resolve => { entered = resolve; }); - const client = core.createTelemetry({ + const client = core.createIngester({ enrichStacktrace: () => { entered(); return gate; @@ -382,7 +382,7 @@ test("reset invalidates active and queued work while allowing immediate new inge }); test("subscribers can reset without a later notification restoring stale state", async () => { - const client = core.createTelemetry(); + const client = core.createIngester(); let reset = false; const observed: number[] = []; client.subscribe(snapshot => { diff --git a/packages/spotlight/src/telemetry/index.ts b/packages/spotlight/src/telemetry/index.ts index b91306258..4da0d70be 100644 --- a/packages/spotlight/src/telemetry/index.ts +++ b/packages/spotlight/src/telemetry/index.ts @@ -6,9 +6,9 @@ import type { Diagnostic, EventFilter, IngestResult, - Telemetry, - TelemetryOptions, - TelemetrySnapshot, + Ingester, + IngesterOptions, + IngesterSnapshot, TraceFilter, } from "./public.ts"; import { @@ -27,7 +27,7 @@ import { generateUuidv4 } from "./uuid.ts"; type RetainedEnvelope = { id: string; envelope: Envelope; receivedAtMs: number }; /** Portable, independently owned telemetry with immutable committed reads. */ -export function createTelemetry(options: TelemetryOptions = {}): Telemetry { +export function createIngester(options: IngesterOptions = {}): Ingester { const maxEnvelopes = options.retention?.maxEnvelopes ?? 500; const maxAgeMs = options.retention?.maxAgeMs; if (!Number.isSafeInteger(maxEnvelopes) || maxEnvelopes < 1) @@ -41,7 +41,7 @@ export function createTelemetry(options: TelemetryOptions = {}): Telemetry { let journal: RetainedEnvelope[] = []; let timer: ReturnType | undefined; const identities = new WeakMap(); - const listeners = new Set<(snapshot: TelemetrySnapshot) => void>(); + const listeners = new Set<(snapshot: IngesterSnapshot) => void>(); let enrichmentCache = new Map(); let currentDiagnostics: Diagnostic[] = []; function diagnostic(item: Diagnostic) { @@ -96,7 +96,7 @@ export function createTelemetry(options: TelemetryOptions = {}): Telemetry { return readonlyCopy({ status: "invalid" as const, diagnostics: [item] }); } let engine = makeEngine(); - function stateSnapshot(): TelemetrySnapshot { + function stateSnapshot(): IngesterSnapshot { const state = engine.store.getState(); return readonlyCopy({ revision, @@ -117,7 +117,7 @@ export function createTelemetry(options: TelemetryOptions = {}): Telemetry { try { listener(committed); } catch { - diagnostic({ code: "subscriber_failed", message: "A Telemetry subscription callback failed." }); + diagnostic({ code: "subscriber_failed", message: "A telemetry subscription callback failed." }); } if (snapshot !== committed) break; } @@ -177,7 +177,7 @@ export function createTelemetry(options: TelemetryOptions = {}): Telemetry { function filteredTraces(filter?: TraceFilter) { return snapshot.traces.filter(trace => filter?.traceId === undefined || trace.trace_id === filter.traceId); } - const client: Telemetry = { + const client: Ingester = { ingestEnvelope: input => { if (disposed) return Promise.resolve(ignored("disposed")); if (!validateEnvelope(input)) @@ -267,11 +267,11 @@ export function createTelemetry(options: TelemetryOptions = {}): Telemetry { getEvents: filter => (filter ? Object.freeze(filterEvents(filter)) : snapshot.events), getErrors: filter => Object.freeze(filterEvents(filter).filter(event => isErrorEvent(event as SentryEvent))) as ReturnType< - Telemetry["getErrors"] + Ingester["getErrors"] >, getTrace: id => snapshot.traces.find(trace => trace.trace_id === id), getTraces: filter => (filter ? Object.freeze(filteredTraces(filter)) : snapshot.traces), - getSpans: filter => Object.freeze(getSpans(filteredTraces(filter) as Trace[])) as ReturnType, + getSpans: filter => Object.freeze(getSpans(filteredTraces(filter) as Trace[])) as ReturnType, getLog: id => snapshot.logs.find(log => log.id === id), getLogs: filter => filter?.traceId === undefined @@ -337,4 +337,4 @@ export function createTelemetry(options: TelemetryOptions = {}): Telemetry { return client; } -export type { Telemetry, TelemetryOptions, TelemetrySnapshot, IngestResult, Diagnostic } from "./public.ts"; +export type { Ingester, IngesterOptions, IngesterSnapshot, IngestResult, Diagnostic } from "./public.ts"; diff --git a/packages/spotlight/src/telemetry/parity.spec.ts b/packages/spotlight/src/telemetry/parity.spec.ts index 0940d8e79..0e2fbf869 100644 --- a/packages/spotlight/src/telemetry/parity.spec.ts +++ b/packages/spotlight/src/telemetry/parity.spec.ts @@ -2,14 +2,14 @@ import { readFileSync } from "node:fs"; import type { Envelope } from "@sentry/core"; import { expect, test } from "vitest"; -import { createTelemetry } from "../sdk/core.ts"; -import { createNodeTelemetry } from "../sdk/node.ts"; +import { createIngester } from "../sdk/core.ts"; +import { createPipeline } from "../sdk/node.ts"; import useSentryStore from "../ui/telemetry/store/store"; import { getAITraces, getQuerySummaries, getSpans } from "./selectors/index.ts"; test("the UI store, browser processor, and Node SDK derive identical traces and queries", async () => { - const spotlight = createNodeTelemetry(); - const browser = createTelemetry(); + const spotlight = createPipeline(); + const browser = createIngester(); const result = await spotlight.ingest({ body: readFileSync(new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url)), contentType: "application/x-sentry-envelope", @@ -30,8 +30,8 @@ test("the UI store, browser processor, and Node SDK derive identical traces and }); test("the UI, browser processor, and Node SDK share log, profile, and AI processing", async () => { - const spotlight = createNodeTelemetry(); - const browser = createTelemetry(); + const spotlight = createPipeline(); + const browser = createIngester(); useSentryStore.getState().resetData(); for (const file of ["continuous_profiling/transaction.txt", "continuous_profiling/profile_chunk_1.txt"]) { const result = await spotlight.ingest({ diff --git a/packages/spotlight/src/telemetry/public.ts b/packages/spotlight/src/telemetry/public.ts index f4a8a3767..bf80912e8 100644 --- a/packages/spotlight/src/telemetry/public.ts +++ b/packages/spotlight/src/telemetry/public.ts @@ -59,12 +59,12 @@ export type RetentionOptions = { /** Optional maximum age since receipt. Expiration runs automatically; dispose releases the timer. */ maxAgeMs?: number; }; -export type TelemetryOptions = { +export type IngesterOptions = { retention?: RetentionOptions; enrichStacktrace?: (event: SentryErrorEvent) => Promise; onDiagnostic?: (diagnostic: Diagnostic) => void; }; -export type TelemetrySnapshot = { +export type IngesterSnapshot = { readonly revision: number; readonly envelopes: readonly DeepReadonly[]; readonly events: readonly ProcessedEvent[]; @@ -73,9 +73,9 @@ export type TelemetrySnapshot = { readonly sdks: readonly DeepReadonly[]; readonly profiles: ReadonlyMap; }; -export type Telemetry = { +export type Ingester = { ingestEnvelope: (input: unknown) => Promise; - getSnapshot: () => TelemetrySnapshot; + getSnapshot: () => IngesterSnapshot; getEnvelope: (id: string) => DeepReadonly | undefined; getEnvelopes: () => readonly DeepReadonly[]; getEvent: (id: string) => ProcessedEvent | undefined; @@ -93,7 +93,7 @@ export type Telemetry = { getAITraces: (filter?: TraceFilter) => readonly AITrace[]; getWebVitals: () => readonly DeepReadonly[]; getSdks: () => readonly DeepReadonly[]; - subscribe: (listener: (snapshot: TelemetrySnapshot) => void) => () => void; + subscribe: (listener: (snapshot: IngesterSnapshot) => void) => () => void; reset: () => void; prune: () => Promise; dispose: () => void; diff --git a/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx b/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx index 684ee46bb..88f3c8c0e 100644 --- a/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx +++ b/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx @@ -1,12 +1,12 @@ import { cleanup, render, screen } from "@testing-library/react"; import { afterEach, expect, test } from "vitest"; -import { createTelemetry } from "../../../../telemetry/index.ts"; +import { createIngester } from "../../../../telemetry/index.ts"; import EventContexts from "./EventContexts"; afterEach(cleanup); test("renders modules alongside extra data without mutating the SDK event", async () => { - const telemetry = createTelemetry(); + const telemetry = createIngester(); await telemetry.ingestEnvelope([ {}, [ diff --git a/packages/spotlight/src/ui/telemetry/store/store.ts b/packages/spotlight/src/ui/telemetry/store/store.ts index a2ead4830..af6b99432 100644 --- a/packages/spotlight/src/ui/telemetry/store/store.ts +++ b/packages/spotlight/src/ui/telemetry/store/store.ts @@ -2,8 +2,8 @@ import type { Envelope } from "@sentry/core"; import { CONTEXT_LINES_ENDPOINT } from "@spotlight/shared/constants.ts"; import { DEFAULT_SIDECAR_URL } from "@spotlight/ui/constants"; import { useStore } from "zustand"; -import { createTelemetry } from "../../../sdk/core.ts"; -import type { IngestResult, TelemetrySnapshot } from "../../../sdk/core.ts"; +import { createIngester } from "../../../sdk/core.ts"; +import type { IngestResult, IngesterSnapshot } from "../../../sdk/core.ts"; import type { SentryStore, SentryStoreState, Subscription } from "../../../telemetry/store/types.ts"; import { getNativeFetchImplementation } from "../utils/fetch"; @@ -12,7 +12,7 @@ export function setTelemetrySidecarUrl(url: string) { contextLinesProvider = new URL(CONTEXT_LINES_ENDPOINT, url).href; } -const spotlight = createTelemetry({ +const spotlight = createIngester({ enrichStacktrace: async event => { await Promise.all( (event.exception?.values ?? []).map(async exception => { @@ -95,8 +95,8 @@ const actions = { }; }, }; -function view(snapshot: TelemetrySnapshot): UiState { - const logsByTraceId = new Map>(); +function view(snapshot: IngesterSnapshot): UiState { + const logsByTraceId = new Map>(); for (const log of snapshot.logs) { if (!log.trace_id) continue; const group = logsByTraceId.get(log.trace_id) ?? new Set(); diff --git a/packages/spotlight/tests/types/sdk-core.ts b/packages/spotlight/tests/types/sdk-core.ts index f3a08d42c..674e540e2 100644 --- a/packages/spotlight/tests/types/sdk-core.ts +++ b/packages/spotlight/tests/types/sdk-core.ts @@ -1,6 +1,6 @@ -import { type ProcessedEvent, type QuerySummary, createTelemetry } from "@spotlightjs/spotlight/sdk/core"; +import { type ProcessedEvent, type QuerySummary, createIngester } from "@spotlightjs/spotlight/sdk/core"; -const telemetry = createTelemetry(); +const telemetry = createIngester(); const queries: readonly QuerySummary[] = telemetry.getQuerySummaries({ traceId: "trace" }); const events: readonly ProcessedEvent[] = telemetry.getEvents({ traceId: "trace", spanId: "span" }); const envelope = telemetry.getEnvelope("envelope"); From 3028195cf063f31e91ece85574f5e99347166fcd Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Tue, 27 Oct 2026 15:39:24 +0530 Subject: [PATCH 3/7] feat(sdk): Add bounded ingestion and agent-friendly reads --- .github/workflows/build.yml | 3 + README.md | 6 + package.json | 1 + packages/spotlight/README.md | 250 ++++---- packages/spotlight/examples/sdk/frontend.ts | 4 +- packages/spotlight/examples/sdk/lifecycle.ts | 82 +++ packages/spotlight/examples/sdk/react.tsx | 15 +- packages/spotlight/examples/sdk/reader.ts | 31 + packages/spotlight/examples/sdk/server.ts | 24 +- packages/spotlight/examples/sdk/tsconfig.json | 14 + packages/spotlight/package.json | 1 + packages/spotlight/src/sdk/core.ts | 36 ++ packages/spotlight/src/sdk/node.spec.ts | 546 ++++++++++++++-- packages/spotlight/src/sdk/node.ts | 433 ++++++++++--- .../src/server/formatters/human/utils.ts | 154 +---- .../src/server/parser/processEnvelope.ts | 158 +---- .../src/server/routes/stream/userAgent.ts | 58 +- packages/spotlight/src/server/sdk.spec.ts | 12 +- .../spotlight/src/telemetry/capabilities.ts | 18 + .../spotlight/src/telemetry/client.spec.ts | 2 +- packages/spotlight/src/telemetry/contracts.ts | 76 +++ .../src/telemetry/envelope/codec.spec.ts | 62 ++ .../spotlight/src/telemetry/envelope/codec.ts | 137 ++++ .../src/telemetry/envelope/payload.ts | 24 + .../src/telemetry/envelope/source.ts | 153 +++++ .../src/telemetry/envelope/userAgent.ts | 57 ++ packages/spotlight/src/telemetry/index.ts | 594 ++++++++++++------ .../src/telemetry/internal/envelopes.ts | 29 +- .../src/telemetry/internal/results.ts | 35 ++ .../spotlight/src/telemetry/lifecycle.spec.ts | 231 +++++++ .../spotlight/src/telemetry/parity.spec.ts | 6 +- packages/spotlight/src/telemetry/public.ts | 73 ++- .../spotlight/src/telemetry/reader.spec.ts | 180 ++++++ packages/spotlight/src/telemetry/reader.ts | 407 ++++++++++++ .../src/telemetry/serialization.spec.ts | 214 +++++++ .../spotlight/src/telemetry/serialization.ts | 150 +++++ .../spotlight/src/ui/telemetry/store/store.ts | 8 +- packages/spotlight/tests/types/sdk-core.ts | 73 ++- packages/spotlight/tests/types/sdk-node.ts | 29 + .../tests/types/tsconfig.sdk-node.json | 5 + scripts/check-sdk-types.mjs | 41 ++ 41 files changed, 3513 insertions(+), 919 deletions(-) create mode 100644 packages/spotlight/examples/sdk/lifecycle.ts create mode 100644 packages/spotlight/examples/sdk/reader.ts create mode 100644 packages/spotlight/examples/sdk/tsconfig.json create mode 100644 packages/spotlight/src/telemetry/capabilities.ts create mode 100644 packages/spotlight/src/telemetry/contracts.ts create mode 100644 packages/spotlight/src/telemetry/envelope/codec.spec.ts create mode 100644 packages/spotlight/src/telemetry/envelope/codec.ts create mode 100644 packages/spotlight/src/telemetry/envelope/payload.ts create mode 100644 packages/spotlight/src/telemetry/envelope/source.ts create mode 100644 packages/spotlight/src/telemetry/envelope/userAgent.ts create mode 100644 packages/spotlight/src/telemetry/internal/results.ts create mode 100644 packages/spotlight/src/telemetry/lifecycle.spec.ts create mode 100644 packages/spotlight/src/telemetry/reader.spec.ts create mode 100644 packages/spotlight/src/telemetry/reader.ts create mode 100644 packages/spotlight/src/telemetry/serialization.spec.ts create mode 100644 packages/spotlight/src/telemetry/serialization.ts create mode 100644 packages/spotlight/tests/types/sdk-node.ts create mode 100644 packages/spotlight/tests/types/tsconfig.sdk-node.json create mode 100644 scripts/check-sdk-types.mjs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 28b6ef5a7..e96743558 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,6 +113,9 @@ jobs: FOSSILIZE_PLATFORMS: linux-x64,linux-arm64,win-x64,darwin-x64,darwin-arm64 FOSSILIZE_SIGN: ${{ github.event_name == 'push' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) && 'y' || 'n' }} run: pnpm build + + - name: Check built SDK types and run examples + run: pnpm test:sdk-types - name: Checking npx run: npx ./packages/spotlight --help diff --git a/README.md b/README.md index 878873207..bfba4bd5f 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,12 @@ Spotlight offers two main ways to work with your development telemetry: Spotlight uses Sentry SDKs to capture telemetry data and sends it to the Sidecar, a local proxy server running on your machine. The Sidecar processes these events and forwards them to the UI, where you can view everything in real-time via the web interface or stream output to your terminal. Spotlight works standalone or alongside your existing Sentry setup without interference. +## Embed the telemetry SDK + +Use `createIngester()` from `@spotlightjs/spotlight/sdk/core` to process parsed Sentry envelopes in portable runtimes, or `createPipeline()` from `@spotlightjs/spotlight/sdk/node` for serialized and compressed Node payloads. Both share the same processing, receipts, readers, and retention controls. + +See the [SDK reference](packages/spotlight/README.md) and [executable examples](packages/spotlight/examples/sdk) for integration, retries, cancellation, paging, JSON exports, and React subscriptions. + ## Links - [![Documentation](https://img.shields.io/badge/documentation-spotlight-indigo.svg)](https://spotlightjs.com/about/) diff --git a/package.json b/package.json index 135271b8b..cf8e74cce 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "dev:website": "turbo dev --filter=@spotlightjs/website", "dev:electron": "pnpm run --filter=@spotlightjs/spotlight dev:electron", "build": "turbo build", + "test:sdk-types": "pnpm --filter @spotlightjs/spotlight test:sdk-types", "preview": "turbo preview", "lint": "biome check .", "lint:fix": "biome check --write .", diff --git a/packages/spotlight/README.md b/packages/spotlight/README.md index 1e6e473b8..b63ccb23e 100644 --- a/packages/spotlight/README.md +++ b/packages/spotlight/README.md @@ -1,166 +1,164 @@ # `@spotlightjs/spotlight` Spotlight provides a development telemetry UI, a local sidecar, and a desktop app. -Its SDK shares the same normalization, trace assembly, and insight calculations -across server and frontend consumers. +Its SDK shares normalization, trace assembly, and insight calculations across +server and frontend consumers. Each instance owns its state; feed server and +frontend instances the same envelope stream and match retention settings when +comparing their retained histories. ## Choose an entry point | Entry | Factory | Input | | --- | --- | --- | -| `@spotlightjs/spotlight/sdk/core` | `createIngester()` | Parsed Sentry envelopes; portable across server and frontend | -| `@spotlightjs/spotlight/sdk/node` | `createPipeline()` | Adds serialized bodies, decompression, and raw payload history | +| `@spotlightjs/spotlight/sdk/core` | `createIngester()` | Parsed Sentry envelopes in portable runtimes | +| `@spotlightjs/spotlight/sdk/node` | `createPipeline()` | Adds serialized bodies, decompression, and decoded payload history | | `@spotlightjs/spotlight/sdk` | Existing buffer helpers | Compatibility for existing transport consumers | -`createIngester()` runs on the frontend or server with parsed envelopes. -`createPipeline()` requires Node APIs for wire decoding and decompression. -Both share all processing and query logic. The core requires no React, DOM, -Node globals, or Zustand knowledge. Each instance owns its state. The SDK does not start a server or synchronize instances; feed -server and frontend instances the same envelope stream when both need processing. -Match retention settings when comparing their retained histories. +## Ingest and read telemetry -## Server: receive envelopes and read insights +Use `@spotlightjs/spotlight/sdk/core` for parsed Sentry envelopes in a browser, worker, or Node application. Use `@spotlightjs/spotlight/sdk/node` for serialized or compressed payloads on Node 20+. Both expose the same processing and read APIs; creating an instance does not start a server. The portable entry has no Node or React dependency. The existing raw-buffer helpers at `/sdk` remain available. ```ts -import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; +import { createIngester, serializeSnapshot } from "@spotlightjs/spotlight/sdk/core"; -const pipeline = createPipeline({ - rawBufferSize: 100, - retention: { maxEnvelopes: 1000, maxAgeMs: 10 * 60 * 1000 }, +const telemetry = createIngester(); +const receipt = await telemetry.ingestEnvelope(parsedEnvelope, { + deliveryId: "transport-request-123", // Reuse for retries of this delivery. + signal: abortController.signal, }); +if (receipt.status === "processed") { + console.log(receipt.envelopeId, receipt.counts, receipt.retained); +} else if (receipt.status === "ignored") { + console.log(receipt.reason); // For example: duplicate, aborted, queue_full. +} else { + console.log(receipt.diagnostics); // Literal codes, severity, optional itemIndex/spanIndex. +} +await telemetry.drain(); +const json = JSON.stringify(serializeSnapshot(telemetry.getSnapshot())); +telemetry.dispose(); +``` -const result = await pipeline.ingest({ - body: Buffer.from(envelopeBody), - // Defaults to application/x-sentry-envelope. - // contentType: request.headers["content-type"], - // encoding: request.headers["content-encoding"], // identity, gzip, deflate, br - // userAgent: request.headers["user-agent"], -}); +The Node adapter accepts `string`, `Uint8Array`, and `Buffer` bodies. `ingestPayload({ body, contentType?, encoding?, userAgent?, deliveryId?, signal? })` decodes `application/x-sentry-envelope` (the default content type) and supports identity, gzip, deflate, and Brotli encoding. Use `getDecodedPayload(envelopeId)` for a copied decompressed payload when it remains in the independently bounded history. Parsed envelopes still use `ingestEnvelope`; the payload methods are `ingestPayload` and `ingestPayloadBatch`. + +```ts +import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; -if (result.status === "invalid") { - console.error(result.diagnostics); -} else if (result.status === "processed") { - const trace = pipeline.getTrace(traceId); - const queries = pipeline.getQuerySummaries({ traceId }); - const raw = pipeline.getRawPayload(result.envelopeId); // defensive byte copy +const pipeline = createPipeline({ rawBufferSize: 100 }); +const receipt = await pipeline.ingestPayload({ body: envelopeBody, deliveryId: "request-1" }); +if (receipt.status === "processed") { + const payload = pipeline.getDecodedPayload(receipt.envelopeId); // Defensive decoded-byte copy. + const queries = pipeline.getQuerySummaries({ traceId }); // Compact by default. } +await pipeline.drain(); +pipeline.dispose(); ``` -See [the HTTP server example](examples/sdk/server.ts), which receives envelopes -at `POST /envelope` and serves query summaries at `GET /queries?traceId=...`. -Call `server.listen(...)` to start that example; closing it disposes its SDK instance. +### Receipts, retries, and lifecycle -## Frontend: process the same stream +Every input returns an immutable receipt with `status`, `diagnostics`, `revision`, `retained`, and `counts`: -```ts -import { createIngester } from "@spotlightjs/spotlight/sdk/core"; +| Field | Meaning | +| --- | --- | +| `status: "processed"` | A supported envelope was processed. This does not imply new telemetry or continued retention. | +| `status: "ignored"` | `reason` explains the outcome: `duplicate`, `unsupported_items`, `unsupported_content_type`, `reset`, `disposed`, `aborted`, or `queue_full`. | +| `status: "invalid"` | Diagnostic codes explain invalid data or exceeded payload limits. | +| `retained` | Whether this envelope is in processed retention when the receipt is produced. An unsupported envelope can be retained; an immediately evicted processed envelope is not. This says nothing about Node decoded history. | +| `revision` | Published snapshot revision at the outcome. A retry that changes no state need not advance it. | +| `counts.events`, `counts.spans`, `counts.logs` | Newly added event IDs, trace/span pairs, and log IDs during processing, before retention eviction. These overlap: standalone spans can also become events. They are not retained collection sizes. | +| `counts.skippedItems` | Unsupported wire items plus malformed entries skipped inside supported span containers. | +| `counts.duplicateEvents` | Candidate event observations that did not add a new event ID. Delivery-level duplicate receipts have zero counts. | -const ingester = createIngester(); +Branch on diagnostic `code` and `severity`, or ignored `reason`, rather than message text. Diagnostics can accompany successful partial processing. `itemIndex` and `spanIndex`, when present, are zero-based locations. -// Your SSE/WebSocket handler supplies parsed envelopes. -await ingester.ingestEnvelope(parsedEnvelope); +Delivery IDs deduplicate only while the corresponding envelope is retained. Eviction and `reset()` permit that delivery again. There is no content hashing: separately allocated envelopes without IDs are separate observations. Reusing the same parsed object or its explicit Spotlight envelope ID preserves its delivery identity within the retained window. Event-ID deduplication is separate from delivery deduplication. -const queries = ingester.getQuerySummaries({ traceId }); -// [{ description, count, totalDurationMs, averageDurationMs, spans }] -const logs = ingester.getLogs({ traceId }); -const spans = ingester.getSpans({ traceId }); -``` +`ingestBatch(iterable, { signal? })` returns receipts in input order and preserves valid neighbors of invalid inputs. It is not a transaction. `ingestPayloadBatch(inputs)` does the same for Node inputs; each input carries its own signal and delivery ID. A batch occupies one queued operation, including while active. Items are copied as they are reached rather than copying the entire batch up front: keep queued batch items unchanged until their receipts resolve. Individual `ingestEnvelope` and `ingestPayload` calls copy their input before the first await. Cancellation returns `ignored/aborted`. `drain()` waits for operations already submitted at the call, excluding future work. `getStats()` reports pending operations, retained envelope count/bytes, revision, and disposal. `subscribe((snapshot, change) => ...)` returns an unsubscribe function; `change` identifies ingest, eviction, reset, or disposal and affected envelope IDs. Release timers and listeners with `dispose()`. -See [the frontend lifecycle example](examples/sdk/frontend.ts). Create an instance -once for the view or application, then call `dispose()` when it is no longer needed. +An optional `enrichStacktrace(event, { signal })` hook receives a copied event. Observe its signal to cancel external work. A timeout reports a warning and keeps telemetry; abort, reset, disposal, or timeout prevent late hook writes from entering published state. -## React: subscribe to completed snapshots +### Read APIs and JSON -```tsx -import { useSyncExternalStore } from "react"; -import type { Ingester } from "@spotlightjs/spotlight/sdk/core"; +Accept a `TelemetryReader` from `/sdk/core` in consumers that only query telemetry. Both factories implement it. `createTelemetryReader(getSnapshot)` also constructs a reader over an existing snapshot source. Singular getters such as `getTrace(id)` and `getEvent(id)` return `undefined` when absent; collection getters return readonly arrays. All processed timestamps and durations use milliseconds; time bounds are inclusive Unix milliseconds. -function TraceCount({ ingester }: { ingester: Ingester }) { - const snapshot = useSyncExternalStore( - ingester.subscribe, - ingester.getSnapshot, - ingester.getSnapshot, - ); - return {snapshot.traces.length} traces; +```ts +const queries = reader.getQuerySummaries({ traceId, fromMs, toMs, limit: 10 }); +const detailed = reader.getQuerySummaries({ traceId, includeSpans: true }); +const first = reader.readPage("events", { traceId, limit: 100, order: "desc" }); +if (first.status === "ok" && first.nextCursor) { + const next = reader.readPage("events", { + traceId, limit: 100, order: "desc", cursor: first.nextCursor, + }); + // Handle next.status === "invalid" before using next.items. } ``` -See [the React query-list example](examples/sdk/react.tsx). Snapshots keep the same -identity until a committed update, making them suitable for `useSyncExternalStore`. -The core never exposes a writable store or requires a React adapter dependency. +Queries and resource summaries omit full span lists by default; opt in with `includeSpans: true`. Getters accept `fromMs`, `toMs`, `order` (`asc` by default), and a positive `limit`; getters have no implicit limit. Chronological ties use stable IDs. Summary rows sort by description and apply time bounds to their input spans. Invalid getter options throw `RangeError`. + +`readPage` supports `events`, `traces`, `logs`, and `spans`, defaults to 100 rows, and caps a page at 1000. It returns `status: "ok"`, `revision`, `items`, `total`, and optional `nextCursor`, or `status: "invalid"` with `invalid_query`, `invalid_cursor`, or `stale_cursor`. Cursors bind the collection, filters, order, limit, and snapshot revision. Keep the query unchanged between pages and restart after a stale cursor. `spanId` filtering on pages is supported for events only. Treat cursors as opaque. + +Snapshots preserve readonly maps and sets for in-process use. For JSON, use `serializeSnapshot(snapshot)` or `serializeTrace(trace)`: the schema version is `1`, maps become entry arrays, sets become value arrays, and span trees use IDs and `childSpanIds` instead of repeated object graphs. Unsupported non-wire values or unrelated cycles throw `TypeError`. These DTOs are export formats, not inputs to `ingestEnvelope`. Equivalent reads within the current revision reuse cached results; older snapshots remain stable after subsequent ingestion. + +Use `serializeSummaries(reader.getProfileFunctionSummaries())` or pass compact query/resource summaries to export `{ schemaVersion: 1, summaries }`. Profile `traceIds` sets become string arrays. Summaries requested with `includeSpans: true` are rejected at both the type and runtime boundaries; export the complete span graph with `serializeTrace` instead. + +### Limits and supported formats + +| Option | Default | +| --- | --- | +| `maxPending` (ingestion operations, including active work) | 64 | +| `maxEnvelopeBytes` (serialized parsed envelope) | 32 MiB | +| `enrichmentTimeoutMs` | 5000 ms | +| `retention.maxEnvelopes` / `retention.maxBytes` | 500 / 64 MiB | +| `retention.maxAgeMs` | No age limit; when configured, expires automatically since receipt | +| Node `maxPayloadBytes` / `maxDecodedBytes` | 8 MiB / 32 MiB | +| Node `rawBufferSize` / `maxRawBytes` | 500 / 32 MiB | + +Limits must be positive safe integers; Node `rawBufferSize` and `maxRawBytes` also allow zero to disable decoded history. Byte limits use bytes, with 1 MiB = 1024 × 1024 bytes. The Node decoder bounds decompression output. Bound transport collection separately before passing a complete body to the SDK, as shown in the HTTP example. Retention evicts whole envelopes and the telemetry derived from them; the byte budget measures the serialized retained envelopes, not total JavaScript heap size. + +`SDK_CAPABILITIES` exposes readonly machine-readable metadata for legacy and v2 spans, profile v1/v2, supported item types, and timestamp units. Supported item types include event/error, transaction, span, log, profile/profile_chunk, feedback, and attachment; individual wire versions can still be unsupported. Portable runtimes need standard JavaScript APIs including `AbortController`, `TextEncoder`, timers, maps, and sets. React is required only by your React consumer; compressed payload decoding is provided by the Node entry. -## Reads and units +The [SDK examples](examples/sdk) cover a bounded HTTP server, a frontend transport adapter, a reader-only consumer, React subscription, and an executable lifecycle walkthrough with receipts, retry, partial batches, cancellation, drain, JSON, and paging. After building the Node and UI outputs, run `pnpm test:sdk-types` to typecheck the published declarations and every example, then execute the lifecycle walkthrough. CI runs this check after the package build; it does not add a whole-project TypeScript gate. -Collection reads accept filters; singular reads accept an identity. +### Getter reference | Read | Result | | --- | --- | -| `getTrace(traceId)`, `getTraces({ traceId? })` | Assembled traces | -| `getEvent(eventId)`, `getEvents({ traceId?, spanId? })` | Normalized events | -| `getErrors({ traceId?, spanId? })` | Captured exception events | -| `getSpans({ traceId? })` | Normalized spans | -| `getLog(logId)`, `getLogs({ traceId? })` | Logs and trace correlation | -| `getQuerySummaries({ traceId?, description? })` | Database spans grouped by trimmed description | -| `getResourceSummaries({ traceId? })` | Resource timings and encoded sizes | -| `getProfile(traceId)` | Associated V1 or continuous profile | -| `getProfileFunctionSummaries({ traceId? })` | Inclusive frame durations and sample counts | -| `getAITraces({ traceId? })` | AI interactions, tokens, prompts, responses, tools | -| `getWebVitals()` | Scored page-load events | +| `getTrace(traceId)`, `getTraces(filter?)` | Assembled traces | +| `getEvent(eventId)`, `getEvents(filter?)`, `getErrors(filter?)` | Normalized events and captured exceptions | +| `getSpans(filter?)` | Normalized spans | +| `getLog(logId)`, `getLogs(filter?)` | Logs and trace correlation | +| `getQuerySummaries(filter?)` | Database spans grouped by trimmed description | +| `getResourceSummaries(filter?)` | Resource timings and encoded sizes | +| `getProfile(traceId)`, `getProfileFunctionSummaries(filter?)` | Associated profiles and inclusive frame durations/sample counts | +| `getAITraces(filter?)` | AI interactions, tokens, prompts, responses, and tools | +| `getWebVitals(filter?)` | Scored page-load events | | `getEnvelope(envelopeId)`, `getEnvelopes()` | Retained wire envelopes | | `getSdks()` | Observed SDK identities | -Normalized event/span timestamps are Unix **milliseconds**. Summary fields name -units explicitly: `totalDurationMs`, `averageDurationMs`, `averageEncodedSizeBytes`. -Stored envelopes retain wire timestamp units. Parsed attachment strings and bytes -use the same `{ data }` representation as the Node decoder: UTF-8 for text content -types, base64 for binary data. Internal envelope metadata is added to the stored -copy. Caller input is never mutated. Telemetry reads return read-only snapshots, -arrays, objects, maps, and sets; copy data when you need to edit it. - -## Ingestion results and diagnostics - -Both factories return the same discriminated result from ingestion: - -- `processed`: accepted telemetry, with `envelopeId` and `diagnostics`. -- `ignored`: `reason` explains a duplicate, unsupported item/media type, reset, or - disposed instance. Valid unsupported-only envelopes are retained for inspection - and include `envelopeId`. -- `invalid`: malformed input or decompression failure, with `diagnostics`. - -Mixed span batches retain valid spans and report skipped malformed spans. Supply -`onDiagnostic(diagnostic)` to observe diagnostics, or inspect each ingestion result. -Optional `enrichStacktrace: async event => { ... }` adds source context without -requiring a context-lines server. Enrichment failure retains the captured error. - -Ingestion resolves after processing/enrichment. Each committed envelope produces -one coherent subscription update; duplicate or invalid input produces none unless -retention also removes expired data. Subscribers can read new raw Node payloads -at that same update. A subscriber or diagnostic observer cannot break ingestion. - -## Retention and lifecycle - -- Processed retention defaults to **500 envelopes**, including telemetry derived - from them. Set `retention.maxEnvelopes` to another positive integer. -- Optional `retention.maxAgeMs` measures age since receipt. Expiration runs while - idle, and `await ingester.prune()` can enforce limits explicitly. -- Eviction rebuilds derived state from retained envelopes so traces, logs, profiles, - and insights stay consistent. Enrichment is reused; generated IDs remain stable. - Rebuilding costs CPU proportional to retained telemetry; use smaller limits for - large profiles or high-volume streams. -- Node raw history is independently bounded by `rawBufferSize` (default **500**; - **0** disables it). Raw bytes can outlive processed retention and are copied on read. -- `reset()` clears captured data and raw history, preserves subscriptions, and - invalidates in-flight/queued work. The instance remains usable. -- `dispose()` clears captured data, releases timers and subscriptions, invalidates - pending work, and closes the instance. Later ingestion returns `ignored/disposed`. -- Stable envelope/event/log/span IDs deduplicate delivery. Reusing an idless parsed - envelope object also deduplicates it. Distinct idless log envelopes are separate - observations, even when their content matches. - -## Compatibility and migration - -Released `createSpotlightBuffer`, `pushToSpotlightBuffer`, and `decompressBody` -exports remain available from `/sdk` with their existing behavior. Raw buffering -alone does not invoke processing. Use `/sdk/core` and `/sdk/node` for the new -processing API. Its factory and type names are independent of the package name, -so moving the package later only requires changing import paths. +Event reads accept `traceId` and `spanId`; trace/log/span/profile/summary reads +accept `traceId`; query summaries also accept `description`. The time/order/limit +filters apply to filtered collection reads, not `getEnvelopes()` or `getSdks()`. +Summary fields name units explicitly, including `totalDurationMs`, +`averageDurationMs`, and `averageEncodedSizeBytes`. Stored envelopes retain wire +timestamp units. Caller input is copied; read results are readonly. + +`reset()` clears captured data and decoded history, keeps subscriptions, and +invalidates in-flight/queued work. `prune()` explicitly enforces retention. +`dispose()` closes the instance; later ingestion returns `ignored/disposed`. +Eviction rebuilds derived state from retained envelopes, with cost proportional +to retained telemetry. Choose smaller limits for large profiles or heavy streams. + +### Examples and compatibility + +- [HTTP server](examples/sdk/server.ts): bounded `POST /envelope` input and query summaries at `GET /queries?traceId=...`; call `server.listen(...)` to start it. +- [Frontend adapter](examples/sdk/frontend.ts): one ingester per telemetry view; dispose it when the view disconnects. +- [Reader-only consumer](examples/sdk/reader.ts): filtered queries, JSON serialization, and bounded page traversal. +- [React query list](examples/sdk/react.tsx): `useSyncExternalStore` using a reader and subscription, with no ingestion methods exposed to the view. +- [Executable lifecycle walkthrough](examples/sdk/lifecycle.ts): checked and run by `pnpm test:sdk-types` after the package build. + +Snapshots retain the same identity until a committed update, suitable for +`useSyncExternalStore`. Subscribers see coherent snapshots; a failing subscriber +or diagnostic observer cannot break ingestion. Released `createSpotlightBuffer`, +`pushToSpotlightBuffer`, and `decompressBody` remain available from `/sdk` with +their existing behavior. Raw buffering alone does not invoke processing. + +Pruning uses one coalesced control operation outside ingestion admission; it remains part of the drain barrier. diff --git a/packages/spotlight/examples/sdk/frontend.ts b/packages/spotlight/examples/sdk/frontend.ts index 7e2662a6c..e516b035f 100644 --- a/packages/spotlight/examples/sdk/frontend.ts +++ b/packages/spotlight/examples/sdk/frontend.ts @@ -1,4 +1,4 @@ -import { createIngester } from "@spotlightjs/spotlight/sdk/core"; +import { type IngestOptions, createIngester } from "@spotlightjs/spotlight/sdk/core"; /** Keep one instance for the lifetime of the telemetry view. */ export function createFrontendTelemetry() { @@ -6,7 +6,7 @@ export function createFrontendTelemetry() { return { ingester, // Pass the parsed envelopes received over SSE, WebSocket, or your own transport. - receiveEnvelope: (envelope: unknown) => ingester.ingestEnvelope(envelope), + receiveEnvelope: (envelope: unknown, options?: IngestOptions) => ingester.ingestEnvelope(envelope, options), disconnect: ingester.dispose, }; } diff --git a/packages/spotlight/examples/sdk/lifecycle.ts b/packages/spotlight/examples/sdk/lifecycle.ts new file mode 100644 index 000000000..6c06106af --- /dev/null +++ b/packages/spotlight/examples/sdk/lifecycle.ts @@ -0,0 +1,82 @@ +import { type RawEnvelope, SDK_CAPABILITIES } from "@spotlightjs/spotlight/sdk/core"; +import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; +import { createFrontendTelemetry } from "./frontend.js"; +import { inspectTrace, readEventIds } from "./reader.js"; + +// Parsed wire timestamps are seconds; normalized reader timestamps are Unix milliseconds. +function envelope(id: string): RawEnvelope { + return [ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "example-trace", + span_id: id, + name: "SELECT 1", + start_timestamp: 1, + end_timestamp: 1.125, + attributes: { "sentry.op": { type: "string", value: "db.query" } }, + }, + ], + }, + ], + ], + ] as const; +} +function check(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +const { ingester, receiveEnvelope, disconnect } = createFrontendTelemetry(); +const pipeline = createPipeline(); +try { + const input = envelope("one"); + const first = await receiveEnvelope(input, { deliveryId: "request-1" }); + check(first.status === "processed" && first.retained && first.counts.spans === 1, "Receipt counts"); + const retry = await receiveEnvelope(input, { deliveryId: "request-1" }); + check(retry.status === "ignored" && retry.reason === "duplicate", "Retained delivery retry"); + + // Each input gets a receipt. A malformed middle input does not roll back valid neighbors. + const batch = await ingester.ingestBatch([envelope("two"), null, envelope("three")]); + check( + batch[0].status === "processed" && batch[1].status === "invalid" && batch[2].status === "processed", + "Partial batch", + ); + if (batch[1].status === "invalid") { + for (const diagnostic of batch[1].diagnostics) { + console.log(diagnostic.code, diagnostic.severity, diagnostic.itemIndex); + } + } + const controller = new AbortController(); + controller.abort(); + const cancelled = await ingester.ingestEnvelope(envelope("cancelled"), { signal: controller.signal }); + check(cancelled.status === "ignored" && cancelled.reason === "aborted", "Cancellation receipt"); + await ingester.drain(); // A barrier for work already submitted, not future submissions. + check(ingester.getStats().pending === 0, "Drained work"); + + const reader = inspectTrace(ingester, "example-trace"); + check(JSON.parse(reader.snapshotJson).schemaVersion === 1, "JSON schema"); + check(JSON.parse(reader.profileJson).schemaVersion === 1, "Summary JSON schema"); + const page = ingester.readPage("events", { limit: 1 }); + check(page.status === "ok" && page.nextCursor, "Bounded page"); + await receiveEnvelope(envelope("four")); + const stale = ingester.readPage("events", { limit: 1, cursor: page.nextCursor }); + check(stale.status === "invalid" && stale.code === "stale_cursor", "Revision-bound cursor"); + check(readEventIds(ingester).status === "ok", "Reader-only pagination"); + + const wire = (id: string) => `{}\n${JSON.stringify({ type: "span" })}\n${JSON.stringify(envelope(id)[1][0][1])}`; + const payloads = await pipeline.ingestPayloadBatch([{ body: wire("node-one") }, { body: "invalid" }]); + check(payloads[0].status === "processed" && payloads[1].status === "invalid", "Payload batch"); + if (payloads[0].status === "processed") { + check(pipeline.getDecodedPayload(payloads[0].envelopeId)?.body.length, "Copied decoded payload"); + } + await pipeline.drain(); + console.log(SDK_CAPABILITIES.timestampUnit, first.counts, ingester.getStats()); +} finally { + pipeline.dispose(); + disconnect(); +} diff --git a/packages/spotlight/examples/sdk/react.tsx b/packages/spotlight/examples/sdk/react.tsx index 38d672f2b..25c8cc4dd 100644 --- a/packages/spotlight/examples/sdk/react.tsx +++ b/packages/spotlight/examples/sdk/react.tsx @@ -1,9 +1,16 @@ -import type { Ingester } from "@spotlightjs/spotlight/sdk/core"; +import type { Ingester, TelemetryReader } from "@spotlightjs/spotlight/sdk/core"; import { useSyncExternalStore } from "react"; -export function QueryList({ ingester, traceId }: { ingester: Ingester; traceId?: string }) { - useSyncExternalStore(ingester.subscribe, ingester.getSnapshot, ingester.getSnapshot); - const queries = ingester.getQuerySummaries({ traceId }); +type Props = { + reader: TelemetryReader; + subscribe: Ingester["subscribe"]; + traceId?: string; +}; + +/** Pass reader={ingester} subscribe={ingester.subscribe}; the view cannot ingest or reset. */ +export function QueryList({ reader, subscribe, traceId }: Props) { + useSyncExternalStore(subscribe, reader.getSnapshot, reader.getSnapshot); + const queries = reader.getQuerySummaries({ traceId }); return (
      {queries.map(query => ( diff --git a/packages/spotlight/examples/sdk/reader.ts b/packages/spotlight/examples/sdk/reader.ts new file mode 100644 index 000000000..e48a48684 --- /dev/null +++ b/packages/spotlight/examples/sdk/reader.ts @@ -0,0 +1,31 @@ +import { + type TelemetryReader, + serializeSnapshot, + serializeSummaries, + serializeTrace, +} from "@spotlightjs/spotlight/sdk/core"; + +/** Business logic only needs a reader, independent of how telemetry arrives. */ +export function inspectTrace(reader: TelemetryReader, traceId: string) { + const trace = reader.getTrace(traceId); + return { + queries: reader.getQuerySummaries({ traceId, order: "desc", limit: 10 }), + detailedQueries: reader.getQuerySummaries({ traceId, includeSpans: true }), + traceJson: trace ? JSON.stringify(serializeTrace(trace)) : undefined, + snapshotJson: JSON.stringify(serializeSnapshot(reader.getSnapshot())), + profileJson: JSON.stringify(serializeSummaries(reader.getProfileFunctionSummaries({ traceId }))), + }; +} + +/** Keep the filters unchanged across pages; restart after any snapshot change. */ +export function readEventIds(reader: TelemetryReader) { + const ids: string[] = []; + let cursor: string | undefined; + do { + const page = reader.readPage("events", { limit: 1, order: "asc", cursor }); + if (page.status === "invalid") return page; // stale_cursor, invalid_cursor, or invalid_query + ids.push(...page.items.map(event => event.event_id)); + cursor = page.nextCursor; + } while (cursor); + return { status: "ok" as const, ids }; +} diff --git a/packages/spotlight/examples/sdk/server.ts b/packages/spotlight/examples/sdk/server.ts index 5e4a8058f..610d9cf45 100644 --- a/packages/spotlight/examples/sdk/server.ts +++ b/packages/spotlight/examples/sdk/server.ts @@ -1,9 +1,12 @@ import { createServer } from "node:http"; import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; +const maxPayloadBytes = 8 * 1024 * 1024; + /** Embed the SDK in an HTTP server; the SDK itself does not open a port. */ export function createTelemetryServer() { const pipeline = createPipeline({ + maxPayloadBytes, rawBufferSize: 100, retention: { maxEnvelopes: 1000, maxAgeMs: 10 * 60 * 1000 }, }); @@ -13,13 +16,30 @@ export function createTelemetryServer() { response.setHeader("content-type", "application/json"); if (request.method === "POST" && url.pathname === "/envelope") { const chunks: Buffer[] = []; - for await (const chunk of request) chunks.push(Buffer.from(chunk)); - const result = await pipeline.ingest({ + let bytes = 0; + for await (const chunk of request) { + const buffer = Buffer.from(chunk); + bytes += buffer.length; + // Bound HTTP collection too: SDK limits only apply once the body is passed in. + if (bytes > maxPayloadBytes) { + response.writeHead(413).end(JSON.stringify({ error: "Payload too large" })); + return; + } + chunks.push(buffer); + } + const controller = new AbortController(); + const abort = () => controller.abort(); + response.once("close", abort); + const deliveryId = request.headers["idempotency-key"]; + const result = await pipeline.ingestPayload({ body: Buffer.concat(chunks), contentType: request.headers["content-type"], encoding: request.headers["content-encoding"], userAgent: request.headers["user-agent"], + deliveryId: typeof deliveryId === "string" ? deliveryId : undefined, + signal: controller.signal, }); + response.off("close", abort); response.statusCode = result.status === "invalid" ? 400 : 202; response.end(JSON.stringify(result)); return; diff --git a/packages/spotlight/examples/sdk/tsconfig.json b/packages/spotlight/examples/sdk/tsconfig.json new file mode 100644 index 000000000..b712e2234 --- /dev/null +++ b/packages/spotlight/examples/sdk/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023", "DOM"], + "types": ["node", "react"], + "jsx": "react-jsx", + "strict": true, + "skipLibCheck": false, + "rootDir": "." + }, + "include": ["*.ts", "*.tsx"] +} diff --git a/packages/spotlight/package.json b/packages/spotlight/package.json index 30f5217ec..2713d50fb 100644 --- a/packages/spotlight/package.json +++ b/packages/spotlight/package.json @@ -26,6 +26,7 @@ "lint:react-refresh": "eslint 'src/ui/**/*.{ts,tsx}'", "test": "vitest run", "test:dev": "vitest", + "test:sdk-types": "node ../../scripts/check-sdk-types.mjs", "test:e2e": "pnpm run test:e2e:cli && pnpm run test:e2e:ui && pnpm run test:e2e:electron", "test:e2e:cli": "vitest run --config vitest.cli.config.ts", "test:e2e:ui": "playwright test tests/e2e/ui", diff --git a/packages/spotlight/src/sdk/core.ts b/packages/spotlight/src/sdk/core.ts index 2da2fb3b4..bd04f8663 100644 --- a/packages/spotlight/src/sdk/core.ts +++ b/packages/spotlight/src/sdk/core.ts @@ -20,3 +20,39 @@ export type { Diagnostic, IngestResult, } from "../telemetry/public.ts"; + +export { createTelemetryReader } from "../telemetry/reader.ts"; +export type { + TelemetryReader, + TelemetryReadFilter, + TraceReadFilter, + EventReadFilter, + SummaryReadFilter, + DatabaseReadFilter, + CompactQuerySummary, + CompactResourceSummary, + ReadCollection, + ReadCollectionItem, + ReadPageOptions, + ReadPageResult, +} from "../telemetry/reader.ts"; +export { serializeTrace, serializeSnapshot, serializeSummaries } from "../telemetry/serialization.ts"; +export type { + JsonValue, + SerializedSpan, + SerializedTrace, + SerializedSnapshot, + SerializedSummaries, + SerializableSummary, +} from "../telemetry/serialization.ts"; +export { SDK_CAPABILITIES } from "../telemetry/capabilities.ts"; +export type { + DiagnosticCode, + IgnoreReason, + IngestCounts, + ReceiptMetadata, + IngestOptions, + BatchOptions, + ChangeMetadata, + IngestStats, +} from "../telemetry/contracts.ts"; diff --git a/packages/spotlight/src/sdk/node.spec.ts b/packages/spotlight/src/sdk/node.spec.ts index 961e8bf30..4ff407d0b 100644 --- a/packages/spotlight/src/sdk/node.spec.ts +++ b/packages/spotlight/src/sdk/node.spec.ts @@ -17,12 +17,12 @@ describe("Node Telemetry adapter", () => { binary, ]); const telemetry = createPipeline(); - const result = await telemetry.ingest({ body }); + const result = await telemetry.ingestPayload({ body }); expect(result.status).toBe("processed"); expect(telemetry.getEvent("with-opaque-item")).toBeDefined(); expect(telemetry.getEnvelopes()[0][1][1][1]).toEqual({ data: binary.toString("base64") }); if (result.status !== "processed") throw new Error("Expected processed envelope"); - expect(telemetry.getRawPayload(result.envelopeId)?.body).toEqual(body); + expect(telemetry.getDecodedPayload(result.envelopeId)?.body).toEqual(body); telemetry.dispose(); }, ); @@ -31,7 +31,7 @@ describe("Node Telemetry adapter", () => { "rejects a truncated item header without its terminating newline: %s", async body => { const telemetry = createPipeline(); - expect((await telemetry.ingest({ body })).status).toBe("invalid"); + expect((await telemetry.ingestPayload({ body })).status).toBe("invalid"); expect(telemetry.getEnvelopes()).toEqual([]); telemetry.dispose(); }, @@ -39,7 +39,7 @@ describe("Node Telemetry adapter", () => { test("accepts an empty attachment payload after a complete item header", async () => { const telemetry = createPipeline(); - expect((await telemetry.ingest({ body: '{}\n{"type":"attachment","length":0}\n' })).status).toBe("ignored"); + expect((await telemetry.ingestPayload({ body: '{}\n{"type":"attachment","length":0}\n' })).status).toBe("ignored"); expect(telemetry.getEnvelopes()).toHaveLength(1); telemetry.dispose(); }); @@ -52,35 +52,35 @@ describe("Node Telemetry adapter", () => { const observations: (Buffer | undefined)[] = []; spotlight.subscribe(snapshot => { const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); - observations.push(spotlight.getRawPayload(id)?.body); + observations.push(spotlight.getDecodedPayload(id)?.body); }); - await spotlight.ingest({ body }); + await spotlight.ingestPayload({ body }); expect(observations).toEqual([body]); }); test("raw history bounds already apply when subscribers observe the next envelope", async () => { const spotlight = createPipeline({ rawBufferSize: 1 }); - const first = await spotlight.ingest({ body: fixture }); + const first = await spotlight.ingestPayload({ body: fixture }); if (first.status !== "processed") throw new Error("Expected processed envelope"); const observations: { old: Buffer | undefined; newest: Buffer | undefined }[] = []; spotlight.subscribe(snapshot => { const envelope = snapshot.envelopes.find(item => String(item[0].__spotlight_envelope_id) !== first.envelopeId)!; observations.push({ - old: spotlight.getRawPayload(first.envelopeId)?.body, - newest: spotlight.getRawPayload(String(envelope[0].__spotlight_envelope_id))?.body, + old: spotlight.getDecodedPayload(first.envelopeId)?.body, + newest: spotlight.getDecodedPayload(String(envelope[0].__spotlight_envelope_id))?.body, }); }); - await spotlight.ingest({ body: fixture }); + await spotlight.ingestPayload({ body: fixture }); expect(observations).toEqual([{ old: undefined, newest: fixture }]); }); test("defaults to Sentry envelopes and exposes processed raw payloads", async () => { const spotlight = createPipeline(); - const result = await spotlight.ingest({ body: fixture }); + const result = await spotlight.ingestPayload({ body: fixture }); expect(result.status).toBe("processed"); if (result.status !== "processed") throw new Error("Expected processed envelope"); expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); - expect(spotlight.getRawPayload(result.envelopeId)).toEqual({ + expect(spotlight.getDecodedPayload(result.envelopeId)).toEqual({ body: fixture, contentType: "application/x-sentry-envelope", }); @@ -93,7 +93,7 @@ describe("Node Telemetry adapter", () => { ["br", brotliCompressSync], ] as const)("decodes %s payloads", async (encoding, compress) => { const spotlight = createPipeline(); - const result = await spotlight.ingest({ + const result = await spotlight.ingestPayload({ body: new Uint8Array(compress(fixture)), contentType: " Application/X-Sentry-Envelope ; charset=utf-8 ", encoding, @@ -101,17 +101,17 @@ describe("Node Telemetry adapter", () => { expect(result.status).toBe("processed"); expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); if (result.status !== "processed") throw new Error("Expected processed envelope"); - expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual(fixture); }); test("accepts serialized envelope strings", async () => { const spotlight = createPipeline(); - expect((await spotlight.ingest({ body: fixture.toString("utf8") })).status).toBe("processed"); + expect((await spotlight.ingestPayload({ body: fixture.toString("utf8") })).status).toBe("processed"); }); test("returns an explicit ignored result for unsupported content types", async () => { const spotlight = createPipeline(); - expect(await spotlight.ingest({ body: fixture, contentType: "application/json" })).toMatchObject({ + expect(await spotlight.ingestPayload({ body: fixture, contentType: "application/json" })).toMatchObject({ status: "ignored", reason: expect.any(String), }); @@ -121,11 +121,11 @@ describe("Node Telemetry adapter", () => { test("retains raw payloads for valid envelopes containing only unsupported items", async () => { const spotlight = createPipeline(); const body = Buffer.from('{}\n{"type":"attachment","length":5}\nhello'); - const result = await spotlight.ingest({ body }); + const result = await spotlight.ingestPayload({ body }); expect(result).toMatchObject({ status: "ignored", reason: "unsupported_items", envelopeId: expect.any(String) }); if (result.status !== "ignored" || !result.envelopeId) throw new Error("Expected retained unsupported envelope"); expect(spotlight.getEnvelopes()).toHaveLength(1); - expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(body); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual(body); }); test.each([ @@ -152,7 +152,7 @@ describe("Node Telemetry adapter", () => { try { const spotlight = createPipeline(); // Runtime validation is part of the public boundary. - const result = await spotlight.ingest(input as Parameters[0]); + const result = await spotlight.ingestPayload(input as Parameters[0]); expect(result).toMatchObject({ status: "invalid", diagnostics: [{ code }] }); expect(spotlight.getEnvelopes()).toEqual([]); expect(errors).not.toHaveBeenCalled(); @@ -166,57 +166,57 @@ describe("Node Telemetry adapter", () => { test("copies incoming and returned raw payload bytes", async () => { const spotlight = createPipeline(); const body = Buffer.from(fixture); - const pending = spotlight.ingest({ body }); + const pending = spotlight.ingestPayload({ body }); body.fill(0); const result = await pending; expect(result.status).toBe("processed"); if (result.status !== "processed") throw new Error("Expected processed envelope"); - const raw = spotlight.getRawPayload(result.envelopeId)!; + const raw = spotlight.getDecodedPayload(result.envelopeId)!; expect(raw.body).toEqual(fixture); raw.body.fill(0); - expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual(fixture); }); test("bounds raw payload history independently from processed retention", async () => { const spotlight = createPipeline({ rawBufferSize: 1 }); - const first = await spotlight.ingest({ body: fixture }); - const second = await spotlight.ingest({ body: fixture }); + const first = await spotlight.ingestPayload({ body: fixture }); + const second = await spotlight.ingestPayload({ body: fixture }); if (first.status !== "processed" || second.status !== "processed") throw new Error("Expected envelopes"); - expect(spotlight.getRawPayload(first.envelopeId)).toBeUndefined(); - expect(spotlight.getRawPayload(second.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getDecodedPayload(first.envelopeId)).toBeUndefined(); + expect(spotlight.getDecodedPayload(second.envelopeId)?.body).toEqual(fixture); expect(spotlight.getEnvelopes()).toHaveLength(2); spotlight.reset(); - expect(spotlight.getRawPayload(second.envelopeId)).toBeUndefined(); + expect(spotlight.getDecodedPayload(second.envelopeId)).toBeUndefined(); expect(spotlight.getEnvelopes()).toEqual([]); }); test("invalid ingestion cannot evict a retained raw payload", async () => { const spotlight = createPipeline({ rawBufferSize: 1 }); - const first = await spotlight.ingest({ body: fixture }); + const first = await spotlight.ingestPayload({ body: fixture }); if (first.status !== "processed") throw new Error("Expected processed envelope"); - expect((await spotlight.ingest({ body: "not an envelope" })).status).toBe("invalid"); - expect((await spotlight.ingest({ body: "{}\n{}\n{}" })).status).toBe("invalid"); - expect(spotlight.getRawPayload(first.envelopeId)?.body).toEqual(fixture); + expect((await spotlight.ingestPayload({ body: "not an envelope" })).status).toBe("invalid"); + expect((await spotlight.ingestPayload({ body: "{}\n{}\n{}" })).status).toBe("invalid"); + expect(spotlight.getDecodedPayload(first.envelopeId)?.body).toEqual(fixture); expect(spotlight).not.toHaveProperty("buffer"); expect(spotlight).not.toHaveProperty("store"); }); test("retains at most 500 raw payloads by default", async () => { const spotlight = createPipeline({ retention: { maxEnvelopes: 1 } }); - const first = await spotlight.ingest({ body: fixture }); + const first = await spotlight.ingestPayload({ body: fixture }); let last = first; - for (let i = 0; i < 500; i++) last = await spotlight.ingest({ body: fixture }); + for (let i = 0; i < 500; i++) last = await spotlight.ingestPayload({ body: fixture }); if (first.status !== "processed" || last.status !== "processed") throw new Error("Expected envelopes"); - expect(spotlight.getRawPayload(first.envelopeId)).toBeUndefined(); - expect(spotlight.getRawPayload(last.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getDecodedPayload(first.envelopeId)).toBeUndefined(); + expect(spotlight.getDecodedPayload(last.envelopeId)?.body).toEqual(fixture); expect(spotlight.getEnvelopes()).toHaveLength(1); }); test("zero raw buffer capacity still processes telemetry", async () => { const spotlight = createPipeline({ rawBufferSize: 0 }); - const result = await spotlight.ingest({ body: fixture }); + const result = await spotlight.ingestPayload({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); - expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); + expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); }); @@ -224,11 +224,11 @@ describe("Node Telemetry adapter", () => { vi.useFakeTimers(); const spotlight = createPipeline({ retention: { maxAgeMs: 100 } }); try { - const result = await spotlight.ingest({ body: fixture }); + const result = await spotlight.ingestPayload({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); await vi.advanceTimersByTimeAsync(101); expect(spotlight.getEnvelopes()).toEqual([]); - expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual(fixture); } finally { spotlight.dispose(); vi.useRealTimers(); @@ -250,7 +250,7 @@ describe("Node Telemetry adapter", () => { await gate; }, }); - const pending = spotlight.ingest({ + const pending = spotlight.ingestPayload({ body: [ "{}", '{"type":"event"}', @@ -266,7 +266,8 @@ describe("Node Telemetry adapter", () => { release(); const result = await pending; expect(result.status).toBe("ignored"); - if ("envelopeId" in result && result.envelopeId) expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); + if ("envelopeId" in result && result.envelopeId) + expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); expect(spotlight.getEnvelopes()).toEqual([]); }); @@ -277,7 +278,7 @@ describe("Node Telemetry adapter", () => { diagnostics.push(diagnostic.code); }, }); - const result = await spotlight.ingest({ body: fixture, encoding: "zstd" }); + const result = await spotlight.ingestPayload({ body: fixture, encoding: "zstd" }); expect(result.status).toBe("invalid"); expect(diagnostics).toEqual(["unsupported_encoding"]); }); @@ -288,7 +289,7 @@ describe("Node Telemetry adapter", () => { throw new Error("observer failed"); }, }); - expect((await spotlight.ingest({ body: fixture, encoding: "zstd" })).status).toBe("invalid"); + expect((await spotlight.ingestPayload({ body: fixture, encoding: "zstd" })).status).toBe("invalid"); }); test("reset from a subscriber cannot repopulate raw payload history", async () => { @@ -300,19 +301,468 @@ describe("Node Telemetry adapter", () => { spotlight.reset(); } }); - const result = await spotlight.ingest({ body: fixture }); + const result = await spotlight.ingestPayload({ body: fixture }); expect(reset).toBe(true); - if ("envelopeId" in result && result.envelopeId) expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); + if ("envelopeId" in result && result.envelopeId) + expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); expect(spotlight.getEnvelopes()).toEqual([]); }); test("dispose clears raw payloads and prevents subsequent ingestion", async () => { const spotlight = createPipeline(); - const result = await spotlight.ingest({ body: fixture }); + const result = await spotlight.ingestPayload({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); spotlight.dispose(); - expect(spotlight.getRawPayload(result.envelopeId)).toBeUndefined(); - expect((await spotlight.ingest({ body: fixture })).status).toBe("ignored"); + expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); + expect((await spotlight.ingestPayload({ body: fixture })).status).toBe("ignored"); expect(spotlight.getEnvelopes()).toEqual([]); }); }); + +describe("bounded payload ingestion", () => { + test.each([ + ["gzip", gzipSync], + ["deflate", deflateSync], + ["br", brotliCompressSync], + ] as const)("rejects %s expansion at the decompressor output limit", async (encoding, compress) => { + const pipeline = createPipeline({ maxDecodedBytes: 128 }); + const result = await pipeline.ingestPayload({ body: compress(Buffer.alloc(4096, 65)), encoding }); + expect(result).toMatchObject({ + status: "invalid", + retained: false, + diagnostics: [{ code: "decoded_payload_too_large" }], + }); + expect(pipeline.getEnvelopes()).toEqual([]); + pipeline.dispose(); + }); + + test("enforces input and uncompressed decoded byte limits", async () => { + const inputLimited = createPipeline({ maxPayloadBytes: 10 }); + expect(await inputLimited.ingestPayload({ body: fixture })).toMatchObject({ + status: "invalid", + diagnostics: [{ code: "payload_too_large" }], + }); + inputLimited.dispose(); + const decodedLimited = createPipeline({ maxDecodedBytes: 10 }); + expect(await decodedLimited.ingestPayload({ body: fixture })).toMatchObject({ + status: "invalid", + diagnostics: [{ code: "decoded_payload_too_large" }], + }); + decodedLimited.dispose(); + }); + + test("evicts raw bytes by total decoded size before notifying subscribers", async () => { + const pipeline = createPipeline({ maxRawBytes: fixture.length }); + const first = await pipeline.ingestPayload({ body: fixture }); + if (first.status !== "processed") throw new Error("Expected processed envelope"); + const observations: unknown[] = []; + pipeline.subscribe(snapshot => { + const newest = String(snapshot.envelopes.at(-1)![0].__spotlight_envelope_id); + observations.push([pipeline.getDecodedPayload(first.envelopeId), pipeline.getDecodedPayload(newest)?.body]); + }); + const second = await pipeline.ingestPayload({ body: fixture }); + expect(second.status).toBe("processed"); + expect(observations).toEqual([[undefined, fixture]]); + pipeline.dispose(); + }); + + test("zero raw byte budget disables decoded history", async () => { + const pipeline = createPipeline({ maxRawBytes: 0 }); + const result = await pipeline.ingestPayload({ body: fixture }); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + expect(pipeline.getDecodedPayload(result.envelopeId)).toBeUndefined(); + pipeline.dispose(); + }); + + test("forwards delivery identity and abort signals to processing", async () => { + const pipeline = createPipeline(); + const first = await pipeline.ingestPayload({ body: fixture, deliveryId: "wire-delivery" }); + const duplicate = await pipeline.ingestPayload({ body: fixture, deliveryId: "wire-delivery" }); + expect(first.status).toBe("processed"); + expect(duplicate).toMatchObject({ status: "ignored", reason: "duplicate" }); + const controller = new AbortController(); + controller.abort(); + expect(await pipeline.ingestPayload({ body: fixture, signal: controller.signal })).toMatchObject({ + status: "ignored", + reason: "aborted", + }); + expect(pipeline.getEnvelopes()).toHaveLength(1); + pipeline.dispose(); + }); + + test.each(["reset", "dispose"] as const)("%s cancels asynchronous decompression", async action => { + const pipeline = createPipeline(); + const pending = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); + await Promise.resolve(); + pipeline[action](); + expect(await pending).toMatchObject({ status: "ignored", reason: action === "dispose" ? "disposed" : "reset" }); + expect(pipeline.getEnvelopes()).toEqual([]); + pipeline.dispose(); + }); + + test("bounds concurrently admitted payloads and drain includes decoding", async () => { + const pipeline = createPipeline({ maxPending: 1 }); + const first = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); + expect(await pipeline.ingestPayload({ body: fixture })).toMatchObject({ status: "ignored", reason: "queue_full" }); + await pipeline.drain(); + expect((await first).status).toBe("processed"); + expect(pipeline.getEnvelopes()).toHaveLength(1); + pipeline.dispose(); + }); + + test("payload batches retain order and isolate invalid entries", async () => { + const pipeline = createPipeline(); + const results = await pipeline.ingestPayloadBatch([ + { body: fixture, deliveryId: "first" }, + { body: "invalid" }, + { body: fixture, deliveryId: "first" }, + ]); + expect(results.map(result => result.status)).toEqual(["processed", "invalid", "ignored"]); + expect(results[2]).toMatchObject({ reason: "duplicate" }); + pipeline.dispose(); + }); +}); + +describe("payload lifecycle boundaries", () => { + test("delivery IDs key decoded history before public subscriptions", async () => { + const pipeline = createPipeline(); + const observed: unknown[] = []; + pipeline.subscribe(snapshot => { + const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); + observed.push([id, pipeline.getDecodedPayload(id)?.body]); + }); + const result = await pipeline.ingestPayload({ body: fixture, deliveryId: "effective-wire-id" }); + expect(result).toMatchObject({ status: "processed", envelopeId: "effective-wire-id" }); + expect(observed).toEqual([["effective-wire-id", fixture]]); + pipeline.dispose(); + }); + + test("an abort during enrichment releases the payload queue without publishing raw data", async () => { + let start!: () => void; + let release!: () => void; + const started = new Promise(resolve => { + start = resolve; + }); + const gate = new Promise(resolve => { + release = resolve; + }); + const controller = new AbortController(); + const pipeline = createPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const pending = pipeline.ingestPayload({ + body: '{}\n{"type":"event"}\n{"event_id":"aborted-error","timestamp":1,"exception":{"values":[]}}', + signal: controller.signal, + deliveryId: "aborted-wire-id", + }); + await started; + controller.abort(); + expect(await pending).toMatchObject({ status: "ignored", reason: "aborted" }); + expect(pipeline.getDecodedPayload("aborted-wire-id")).toBeUndefined(); + expect((await pipeline.ingestPayload({ body: fixture })).status).toBe("processed"); + release(); + await pipeline.drain(); + expect(pipeline.getEvent("aborted-error")).toBeUndefined(); + pipeline.dispose(); + }); + + test("aborting decompression does not retain or process the decoded output", async () => { + const pipeline = createPipeline(); + const controller = new AbortController(); + const pending = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip", signal: controller.signal }); + await Promise.resolve(); + controller.abort(); + expect(await pending).toMatchObject({ status: "ignored", reason: "aborted" }); + expect(pipeline.getEnvelopes()).toEqual([]); + pipeline.dispose(); + }); + + test.each(["maxPayloadBytes", "maxDecodedBytes", "maxRawBytes", "rawBufferSize"] as const)( + "validates %s configuration", + name => { + for (const invalid of [-1, 0.1, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => createPipeline({ [name]: invalid })).toThrow(RangeError); + } + if (name === "maxPayloadBytes" || name === "maxDecodedBytes") { + expect(() => createPipeline({ [name]: 0 })).toThrow(RangeError); + } + }, + ); +}); + +test("Node pending stats include queued decode work and reset releases admission capacity", async () => { + const pipeline = createPipeline({ maxPending: 1 }); + const old = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); + expect(pipeline.getStats().pending).toBe(1); + await Promise.resolve(); + pipeline.reset(); + expect(pipeline.getStats().pending).toBe(0); + const next = pipeline.ingestPayload({ body: fixture }); + expect(pipeline.getStats().pending).toBe(1); + expect(await old).toMatchObject({ status: "ignored", reason: "reset" }); + expect((await next).status).toBe("processed"); + expect(pipeline.getStats().pending).toBe(0); + pipeline.dispose(); +}); + +test("Node drain excludes payloads admitted after its barrier", async () => { + let start!: () => void; + let release!: () => void; + const started = new Promise(resolve => { + start = resolve; + }); + const gate = new Promise(resolve => { + release = resolve; + }); + const pipeline = createPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const first = pipeline.ingestPayload({ body: fixture }); + const drained = pipeline.drain(); + const later = pipeline.ingestPayload({ + body: '{}\n{"type":"event"}\n{"event_id":"later-error","timestamp":1,"exception":{"values":[]}}', + }); + await started; + await drained; + expect((await first).status).toBe("processed"); + expect(pipeline.getEvent("later-error")).toBeUndefined(); + release(); + expect((await later).status).toBe("processed"); + pipeline.dispose(); +}); + +test("drain includes every entry of an already-submitted payload batch", async () => { + const pipeline = createPipeline({ maxPending: 1 }); + const batch = pipeline.ingestPayloadBatch([ + { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, + { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-second" }, + ]); + await pipeline.drain(); + expect(pipeline.getEnvelopes()).toHaveLength(2); + expect((await batch).map(result => result.status)).toEqual(["processed", "processed"]); + pipeline.dispose(); +}); + +test("reset cancels the remaining entries of a submitted payload batch", async () => { + const pipeline = createPipeline(); + const batch = pipeline.ingestPayloadBatch([ + { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, + { body: fixture, deliveryId: "batch-second" }, + ]); + await Promise.resolve(); + pipeline.reset(); + const results = await batch; + expect(results).toMatchObject([ + { status: "ignored", reason: "reset" }, + { status: "ignored", reason: "reset" }, + ]); + expect(pipeline.getEnvelopes()).toEqual([]); + expect(pipeline.getDecodedPayload("batch-second")).toBeUndefined(); + pipeline.dispose(); +}); + +test("a queued payload batch stays ahead of later payload work", async () => { + let start!: () => void; + let release!: () => void; + const started = new Promise(resolve => { + start = resolve; + }); + const gate = new Promise(resolve => { + release = resolve; + }); + const pipeline = createPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const batch = pipeline.ingestPayloadBatch([ + { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, + { body: fixture, deliveryId: "batch-second" }, + ]); + const drained = pipeline.drain(); + const later = pipeline.ingestPayload({ + body: '{}\n{"type":"event"}\n{"event_id":"later-blocker","timestamp":1,"exception":{"values":[]}}', + }); + await started; + expect(pipeline.getEnvelope("batch-second")).toBeDefined(); + await drained; + expect((await batch).map(result => result.status)).toEqual(["processed", "processed"]); + release(); + await later; + pipeline.dispose(); +}); + +describe("one Node admission queue for every input method", () => { + const parsedEvent = (id: string, error = false) => [ + {}, + [[{ type: "event" }, { event_id: id, timestamp: 1, ...(error ? { exception: { values: [] } } : {}) }]], + ]; + + test.each(["ingestEnvelope", "ingestBatch"] as const)("wire admission bounds later %s calls", async method => { + const pipeline = createPipeline({ maxPending: 1 }); + const wire = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); + const input = parsedEvent("later-parsed"); + const later = + method === "ingestEnvelope" ? await pipeline.ingestEnvelope(input) : (await pipeline.ingestBatch([input]))[0]; + expect(later).toMatchObject({ status: "ignored", reason: "queue_full" }); + expect(pipeline.getStats().pending).toBe(1); + expect((await wire).status).toBe("processed"); + expect(pipeline.getEvent("later-parsed")).toBeUndefined(); + pipeline.dispose(); + }); + + test.each(["ingestPayload", "ingestPayloadBatch"] as const)( + "parsed admission bounds later %s calls", + async method => { + const pipeline = createPipeline({ maxPending: 1 }); + const parsed = pipeline.ingestEnvelope(parsedEvent("first-parsed")); + const input = { body: fixture }; + const later = + method === "ingestPayload" + ? await pipeline.ingestPayload(input) + : (await pipeline.ingestPayloadBatch([input]))[0]; + expect(later).toMatchObject({ status: "ignored", reason: "queue_full" }); + expect(pipeline.getStats().pending).toBe(1); + expect((await parsed).status).toBe("processed"); + pipeline.dispose(); + }, + ); + + test.each(["ingestEnvelope", "ingestBatch"] as const)("wire drain excludes a later gated %s", async method => { + let start!: () => void; + let release!: () => void; + const started = new Promise(resolve => { + start = resolve; + }); + const gate = new Promise(resolve => { + release = resolve; + }); + const pipeline = createPipeline({ + enrichStacktrace: async () => { + start(); + await gate; + }, + }); + const wire = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); + const drained = pipeline.drain(); + const input = parsedEvent("later-gated-parsed", true); + const later = method === "ingestEnvelope" ? pipeline.ingestEnvelope(input) : pipeline.ingestBatch([input]); + await started; + // The wire job must commit first; its already-captured barrier must settle while later work is gated. + expect(pipeline.getQuerySummaries()).toHaveLength(1); + await drained; + expect((await wire).status).toBe("processed"); + release(); + await later; + pipeline.dispose(); + }); + + test("parsed batches retain order ahead of later wire jobs", async () => { + const pipeline = createPipeline(); + const seen: string[] = []; + pipeline.subscribe((_snapshot, change) => { + if (change.type === "ingest") seen.push(...change.envelopeIds); + }); + const batch = pipeline.ingestBatch([ + [{ __spotlight_envelope_id: "parsed-first" }, []], + [{ __spotlight_envelope_id: "parsed-second" }, []], + ]); + const wire = pipeline.ingestPayload({ body: fixture, deliveryId: "wire-third" }); + await pipeline.drain(); + expect(seen).toEqual(["parsed-first", "parsed-second", "wire-third"]); + expect((await batch).map(item => item.status)).toEqual(["ignored", "ignored"]); + expect((await wire).status).toBe("processed"); + pipeline.dispose(); + }); + + test("copied standalone parsed inputs retain weak object identity across calls and batches", async () => { + const pipeline = createPipeline(); + const input = parsedEvent("copied-input"); + const pending = pipeline.ingestEnvelope(input); + input[1] = []; + const first = await pending; + expect(first.status).toBe("processed"); + expect(pipeline.getEvent("copied-input")).toBeDefined(); + const second = await pipeline.ingestEnvelope(input); + expect(second).toMatchObject({ + status: "ignored", + reason: "duplicate", + envelopeId: "envelopeId" in first ? first.envelopeId : "missing", + }); + expect((await pipeline.ingestBatch([input, input])).map(item => item.status)).toEqual(["ignored", "ignored"]); + expect(pipeline.getEnvelopes()).toHaveLength(1); + expect(input[0]).not.toHaveProperty("__spotlight_envelope_id"); + pipeline.dispose(); + }); + + test("prune calls coalesce without exceeding a full shared queue", async () => { + const pipeline = createPipeline({ maxPending: 1 }); + const wire = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); + const first = pipeline.prune(); + const second = pipeline.prune(); + expect(second).toBe(first); + expect(pipeline.getStats().pending).toBe(1); + await Promise.all([wire, first, pipeline.drain()]); + expect(pipeline.getStats().pending).toBe(0); + pipeline.dispose(); + }); +}); + +test("parsed batch clone failures produce per-item receipts and preserve later inputs", async () => { + const pipeline = createPipeline(); + const uncloneable = [{ bad: () => {} }, []]; + const good = [{}, [[{ type: "event" }, { event_id: "after-clone-failure", timestamp: 1 }]]]; + const results = await pipeline.ingestBatch([uncloneable, good, good]); + expect(results).toMatchObject([ + { status: "invalid", diagnostics: [{ code: "invalid_envelope" }] }, + { status: "processed" }, + { status: "ignored", reason: "duplicate" }, + ]); + expect(pipeline.getEnvelopes()).toHaveLength(1); + expect(Object.isFrozen(results[0].diagnostics[0])).toBe(true); + pipeline.dispose(); +}); + +test("reset invalidates queued parsed jobs and the remaining parsed batch entries", async () => { + const pipeline = createPipeline(); + const wire = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); + const parsed = pipeline.ingestEnvelope([{}, []]); + const batch = pipeline.ingestBatch([ + [{}, []], + [{}, []], + ]); + pipeline.reset(); + expect(await wire).toMatchObject({ status: "ignored", reason: "reset" }); + expect(await parsed).toMatchObject({ status: "ignored", reason: "reset" }); + expect(await batch).toMatchObject([ + { status: "ignored", reason: "reset" }, + { status: "ignored", reason: "reset" }, + ]); + expect(pipeline.getStats().pending).toBe(0); + expect(pipeline.getEnvelopes()).toEqual([]); + pipeline.dispose(); +}); + +test("diagnostic callbacks cannot bypass shared admission during core validation", async () => { + const later: Promise[] = []; + const observed: number[] = []; + const pipeline = createPipeline({ + maxPending: 1, + onDiagnostic() { + observed.push(pipeline.getStats().pending); + later.push(pipeline.ingestPayload({ body: fixture })); + }, + }); + const invalid = await pipeline.ingestEnvelope([{}, [[{ type: "span" }, { span_id: "invalid" }]]]); + expect(invalid.status).toBe("invalid"); + expect(observed).toEqual([1]); + expect(await Promise.all(later)).toMatchObject([{ status: "ignored", reason: "queue_full" }]); + pipeline.dispose(); +}); diff --git a/packages/spotlight/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts index 65246c5bb..c55b6673f 100644 --- a/packages/spotlight/src/sdk/node.ts +++ b/packages/spotlight/src/sdk/node.ts @@ -1,16 +1,24 @@ import { Buffer } from "node:buffer"; -import { brotliDecompressSync, gunzipSync, inflateSync } from "node:zlib"; -import { processEnvelope } from "../server/parser/processEnvelope.ts"; -import { type Diagnostic, type IngestResult, type Ingester, type IngesterOptions, createIngester } from "./core.ts"; +import { brotliDecompress, gunzip, inflate } from "node:zlib"; +import type { Envelope } from "@sentry/core"; +import type { BatchOptions, DiagnosticCode, IgnoreReason, IngestOptions } from "../telemetry/contracts.ts"; +import { decodeEnvelope } from "../telemetry/envelope/codec.ts"; +import { normalizeEnvelope, validateEnvelope } from "../telemetry/internal/envelopes.ts"; +import { diagnostic, receipt, reportDiagnostic } from "../telemetry/internal/results.ts"; +import { generateUuidv4 } from "../telemetry/uuid.ts"; +import { type IngestResult, type Ingester, type IngesterOptions, createIngester } from "./core.ts"; export type { IngesterSnapshot as PipelineSnapshot } from "./core.ts"; export type PipelineOptions = IngesterOptions & { - /** - * Maximum decoded raw payloads to retain (default 500); zero disables storage. - * This history is independent of processed telemetry retention and prune(). - */ + /** Maximum retained decoded payloads (default 500); zero disables storage. */ rawBufferSize?: number; + /** Maximum input body size before decompression. Default: 8 MiB. */ + maxPayloadBytes?: number; + /** Maximum body size after decompression. Default: 32 MiB. */ + maxDecodedBytes?: number; + /** Total decoded payload history budget (default 32 MiB); zero disables storage. */ + maxRawBytes?: number; }; export type PipelineInput = { @@ -18,36 +26,64 @@ export type PipelineInput = { contentType?: string; encoding?: string; userAgent?: string; + deliveryId?: string; + signal?: AbortSignal; }; export type RawPayload = { body: Buffer; contentType: string }; export type Pipeline = Ingester & { - ingest(input: PipelineInput): Promise; - getRawPayload(envelopeId: string): RawPayload | undefined; + ingestPayload(input: PipelineInput): Promise; + ingestPayloadBatch(inputs: readonly PipelineInput[]): Promise; + getDecodedPayload(envelopeId: string): RawPayload | undefined; }; const envelopeContentType = "application/x-sentry-envelope"; -const decompressors = new Map Buffer>([ - ["gzip", gunzipSync], - ["deflate", inflateSync], - ["br", brotliDecompressSync], +const decompressors = new Map([ + ["gzip", gunzip], + ["deflate", inflate], + ["br", brotliDecompress], ]); -/** Decode serialized Sentry envelopes with Node APIs, then process them through the shared ingester. */ -export function createPipeline(options: PipelineOptions = {}): Pipeline { - const rawBufferSize = options.rawBufferSize ?? 500; - if (!Number.isSafeInteger(rawBufferSize) || rawBufferSize < 0) { - throw new RangeError("rawBufferSize must be a non-negative safe integer"); +function limit(name: string, value: number, allowZero = false): number { + if (!Number.isSafeInteger(value) || value < (allowZero ? 0 : 1)) { + throw new RangeError(`${name} must be a ${allowZero ? "non-negative" : "positive"} safe integer`); } + return value; +} + +/** Decode bounded serialized Sentry envelopes, then process them through the shared ingester. */ +export function createPipeline(options: PipelineOptions = {}): Pipeline { + const rawBufferSize = limit("rawBufferSize", options.rawBufferSize ?? 500, true); + const maxRawBytes = limit("maxRawBytes", options.maxRawBytes ?? 32 * 1024 * 1024, true); + const maxPayloadBytes = limit("maxPayloadBytes", options.maxPayloadBytes ?? 8 * 1024 * 1024); + const maxDecodedBytes = limit("maxDecodedBytes", options.maxDecodedBytes ?? 32 * 1024 * 1024); + const maxPending = limit("maxPending", options.maxPending ?? 64); + const maxEnvelopeBytes = options.maxEnvelopeBytes ?? 32 * 1024 * 1024; const telemetry = createIngester(options); const rawPayloads = new Map(); const stagedPayloads = new Map(); + const pending = new Set>(); + const decodeControllers = new Set(); + const coreAdmissions = new Set(); + const identities = new WeakMap(); + let rawBytes = 0; let generation = 0; let disposed = false; + let queue = Promise.resolve(); + let pruning: Promise | undefined; function retainRawPayload(envelopeId: string, payload: RawPayload) { + // Oversized individual payloads are processed but cannot enter the raw history. + if (rawBufferSize === 0 || payload.body.length > maxRawBytes || maxRawBytes === 0) return; + const previous = rawPayloads.get(envelopeId); + if (previous) rawBytes -= previous.body.length; rawPayloads.set(envelopeId, payload); - while (rawPayloads.size > rawBufferSize) rawPayloads.delete(rawPayloads.keys().next().value!); + rawBytes += payload.body.length; + while (rawPayloads.size > rawBufferSize || rawBytes > maxRawBytes) { + const oldest = rawPayloads.keys().next().value!; + rawBytes -= rawPayloads.get(oldest)!.body.length; + rawPayloads.delete(oldest); + } } // Register before public listeners so raw history commits with the core snapshot. @@ -61,104 +97,319 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { } }); - function invalid(code: string, message: string): IngestResult { - const diagnostic: Diagnostic = { code, message }; + function ignored(reason: IgnoreReason): IngestResult { + return receipt({ status: "ignored", reason }, { revision: telemetry.getStats().revision }); + } + function invalid(code: DiagnosticCode, message: string): IngestResult { + const item = diagnostic(code, message); + reportDiagnostic(options.onDiagnostic, item); + return receipt({ status: "invalid", diagnostics: [item] }, { revision: telemetry.getStats().revision }); + } + + async function processPayload( + input: PipelineInput, + body: Buffer, + contentType: string, + encoding: string | undefined, + currentGeneration: number, + ) { + function cancelled(): IngestResult | undefined { + if (disposed) return ignored("disposed"); + if (generation !== currentGeneration) return ignored("reset"); + if (input.signal?.aborted) return ignored("aborted"); + return undefined; + } + const initialCancellation = cancelled(); + if (initialCancellation) return initialCancellation; + const decompress = encoding ? decompressors.get(encoding) : undefined; + if (decompress) { + const controller = new AbortController(); + const abort = () => controller.abort(); + decodeControllers.add(controller); + input.signal?.addEventListener("abort", abort, { once: true }); + try { + // zlib applies this bound while producing output, before a compressed bomb can expand in memory. + const decodeOptions = { maxOutputLength: maxDecodedBytes, signal: controller.signal }; + body = await new Promise((resolve, reject) => { + decompress(body, decodeOptions, (error, output) => { + if (error) reject(error); + else resolve(output); + }); + }); + } catch (error) { + const cancellation = cancelled(); + if (cancellation) return cancellation; + if (error && typeof error === "object" && "code" in error && error.code === "ERR_BUFFER_TOO_LARGE") { + return invalid("decoded_payload_too_large", "Decoded envelope exceeds maxDecodedBytes"); + } + return invalid("decompression_failed", "Could not decompress the envelope body"); + } finally { + decodeControllers.delete(controller); + input.signal?.removeEventListener("abort", abort); + } + } + const cancellation = cancelled(); + if (cancellation) return cancellation; + if (body.length > maxDecodedBytes) + return invalid("decoded_payload_too_large", "Decoded envelope exceeds maxDecodedBytes"); + let parseFailed = false; + let envelope: ReturnType = null; try { - options.onDiagnostic?.(diagnostic); + envelope = decodeEnvelope( + body, + input.userAgent, + () => { + parseFailed = true; + }, + true, + ); } catch { - // A diagnostic observer cannot turn invalid input into a rejected ingestion. + parseFailed = true; + } + if (!envelope || parseFailed) return invalid("invalid_envelope", "Malformed serialized Sentry envelope"); + // Normalize the codec's legacy UUID object before staging; the core publishes this exact string ID. + const envelopeId = input.deliveryId ?? String(envelope[0].__spotlight_envelope_id); + const parsedEnvelope = envelope as Envelope; + parsedEnvelope[0].__spotlight_envelope_id = envelopeId; + const payload = { body, contentType }; + if (rawBufferSize > 0 && maxRawBytes > 0 && body.length <= maxRawBytes) stagedPayloads.set(envelopeId, payload); + try { + const result = await withCoreAdmission(() => + telemetry.ingestEnvelope(parsedEnvelope, { + deliveryId: input.deliveryId, + signal: input.signal, + }), + ); + const retainPayload = + result.status === "processed" || (result.status === "ignored" && result.reason === "unsupported_items"); + if ( + retainPayload && + "envelopeId" in result && + result.envelopeId && + !cancelled() && + stagedPayloads.get(envelopeId) === payload + ) { + // Raw history is independent of processed retention, including immediate age/byte eviction. + retainRawPayload(result.envelopeId, payload); + } + return result; + } finally { + if (stagedPayloads.get(envelopeId) === payload) stagedPayloads.delete(envelopeId); } - return { status: "invalid", diagnostics: [diagnostic] }; } - return { - ...telemetry, - async ingest(input) { - if (disposed) return { status: "ignored", reason: "disposed", diagnostics: [] }; - const currentGeneration = generation; - if (!input || (typeof input.body !== "string" && !(input.body instanceof Uint8Array))) { - return invalid("invalid_body", "Envelope body must be a string or Uint8Array"); - } - if (input.contentType !== undefined && typeof input.contentType !== "string") { - return invalid("invalid_content_type", "Content type must be a string"); - } - const contentType = (input.contentType ?? envelopeContentType).split(";")[0].trim().toLowerCase(); - if (contentType !== envelopeContentType) { - return { status: "ignored", reason: "unsupported_content_type", diagnostics: [] }; - } - if (input.encoding !== undefined && typeof input.encoding !== "string") { - return invalid("unsupported_encoding", "Content encoding must be a string"); - } - const encoding = input.encoding?.trim().toLowerCase(); - const decompress = encoding ? decompressors.get(encoding) : undefined; - if (encoding && encoding !== "identity" && !decompress) { - return invalid("unsupported_encoding", `Unsupported content encoding: ${encoding}`); - } + async function withCoreAdmission(work: () => Promise): Promise { + const admission = Symbol(); + coreAdmissions.add(admission); + try { + return await work(); + } finally { + coreAdmissions.delete(admission); + } + } - // Copy before the first await so caller mutations cannot affect parsing or raw history. - let body: Buffer = typeof input.body === "string" ? Buffer.from(input.body, "utf8") : Buffer.from(input.body); - if (decompress) { - try { - body = decompress(body); - } catch { - return invalid("decompression_failed", "Could not decompress the envelope body"); - } + type PreparedEnvelope = { envelope: Envelope; options: IngestOptions }; + function prepareEnvelope(input: unknown, inputOptions: IngestOptions = {}): PreparedEnvelope | IngestResult { + const captured = { deliveryId: inputOptions.deliveryId, signal: inputOptions.signal }; + if (captured.signal?.aborted) return ignored("aborted"); + if (captured.deliveryId !== undefined && (typeof captured.deliveryId !== "string" || !captured.deliveryId.trim())) { + return invalid("invalid_delivery_id", "deliveryId must be a non-empty string."); + } + if (!validateEnvelope(input)) { + return invalid("invalid_envelope", "Expected a parsed envelope [headers, [[itemHeaders, payload], ...]]."); + } + try { + // Clones need an explicit stable ID: the core cannot associate a fresh clone with its original object. + const id = + captured.deliveryId ?? String(input[0].__spotlight_envelope_id ?? identities.get(input) ?? generateUuidv4()); + identities.set(input, id); + const envelope = normalizeEnvelope(input); + envelope[0].__spotlight_envelope_id = id; + if (Buffer.byteLength(JSON.stringify(envelope), "utf8") > maxEnvelopeBytes) { + return invalid("payload_too_large", "Parsed envelope exceeds maxEnvelopeBytes."); } + return { envelope, options: captured }; + } catch { + return invalid("invalid_envelope", "Envelope data must be cloneable wire data."); + } + } - let parseFailed = false; - let parsed: ReturnType = null; - try { - parsed = processEnvelope( - { contentType, data: body }, - input.userAgent, - () => { - parseFailed = true; - }, - true, - ); - } catch { - parseFailed = true; - } - if (!parsed || parseFailed) return invalid("invalid_envelope", "Malformed serialized Sentry envelope"); + function processEnvelope(prepared: PreparedEnvelope, epoch: number): Promise { + if (disposed) return Promise.resolve(ignored("disposed")); + if (epoch !== generation) return Promise.resolve(ignored("reset")); + if (prepared.options.signal?.aborted) return Promise.resolve(ignored("aborted")); + return withCoreAdmission(() => telemetry.ingestEnvelope(prepared.envelope, prepared.options)); + } - const envelopeId = String(parsed.envelope[0].__spotlight_envelope_id); - const payload = { body, contentType }; - if (rawBufferSize > 0) stagedPayloads.set(envelopeId, payload); - try { - const result = await telemetry.ingestEnvelope(parsed.envelope); - const retainPayload = - result.status === "processed" || (result.status === "ignored" && result.reason === "unsupported_items"); - if ( - retainPayload && - "envelopeId" in result && - result.envelopeId && - generation === currentGeneration && - !disposed && - stagedPayloads.get(envelopeId) === payload - ) { - // Age retention can remove an envelope before publication; raw history remains independent. - retainRawPayload(result.envelopeId, payload); + function pendingCount() { + // Core validation may synchronously notify before core admission increments its own count. + // Every admitted Node job still occupies its slot throughout that callback. + return Math.max(pending.size, telemetry.getStats().pending + pending.size - coreAdmissions.size); + } + + type PreparedPayload = { input: PipelineInput; body: Buffer; contentType: string; encoding?: string }; + function preparePayload(input: PipelineInput): PreparedPayload | IngestResult { + if (!input || (typeof input.body !== "string" && !(input.body instanceof Uint8Array))) { + return invalid("invalid_body", "Envelope body must be a string or Uint8Array"); + } + if (input.signal?.aborted) return ignored("aborted"); + if (input.contentType !== undefined && typeof input.contentType !== "string") { + return invalid("invalid_content_type", "Content type must be a string"); + } + const contentType = (input.contentType ?? envelopeContentType).split(";")[0].trim().toLowerCase(); + if (contentType !== envelopeContentType) return ignored("unsupported_content_type"); + if (input.encoding !== undefined && typeof input.encoding !== "string") { + return invalid("unsupported_encoding", "Content encoding must be a string"); + } + const encoding = input.encoding?.trim().toLowerCase(); + if (encoding && encoding !== "identity" && !decompressors.has(encoding)) { + return invalid("unsupported_encoding", `Unsupported content encoding: ${encoding}`); + } + const size = typeof input.body === "string" ? Buffer.byteLength(input.body, "utf8") : input.body.byteLength; + if (size > maxPayloadBytes) return invalid("payload_too_large", "Envelope body exceeds maxPayloadBytes"); + if ((!encoding || encoding === "identity") && size > maxDecodedBytes) { + return invalid("decoded_payload_too_large", "Decoded envelope exceeds maxDecodedBytes"); + } + const body = typeof input.body === "string" ? Buffer.from(input.body, "utf8") : Buffer.from(input.body); + return { input: { ...input }, body, contentType, encoding }; + } + + function enqueue(work: (epoch: number) => Promise): Promise { + const epoch = generation; + const operation = queue.then(() => work(epoch)); + pending.add(operation); + queue = operation.then( + () => {}, + () => {}, + ); + const release = () => { + pending.delete(operation); + }; + void operation.then(release, release); + return operation; + } + + function ingestPayload(input: PipelineInput): Promise { + if (disposed) return Promise.resolve(ignored("disposed")); + if (input?.signal?.aborted) return Promise.resolve(ignored("aborted")); + if (pendingCount() >= maxPending) return Promise.resolve(ignored("queue_full")); + // Standalone payload bytes and metadata are copied before the first await. + const prepared = preparePayload(input); + if ("status" in prepared) return Promise.resolve(prepared); + return enqueue(epoch => + processPayload(prepared.input, prepared.body, prepared.contentType, prepared.encoding, epoch), + ); + } + + function ingestEnvelope(input: unknown, inputOptions: IngestOptions = {}): Promise { + if (disposed) return Promise.resolve(ignored("disposed")); + if (inputOptions.signal?.aborted) return Promise.resolve(ignored("aborted")); + if (pendingCount() >= maxPending) return Promise.resolve(ignored("queue_full")); + // Snapshot standalone parsed data before enqueueing while retaining the original object's delivery identity. + const prepared = prepareEnvelope(input, inputOptions); + if ("status" in prepared) return Promise.resolve(prepared); + return enqueue(epoch => processEnvelope(prepared, epoch)); + } + + function ingestBatch(inputs: Iterable, batchOptions: BatchOptions = {}): Promise { + const signal = batchOptions.signal; + const rejected = disposed ? "disposed" : pendingCount() >= maxPending ? "queue_full" : undefined; + if (rejected) return Promise.resolve(Object.freeze(Array.from(inputs, () => ignored(rejected)))); + return enqueue(async epoch => { + const results: IngestResult[] = []; + for (const input of inputs) { + if (disposed) results.push(ignored("disposed")); + else if (epoch !== generation) results.push(ignored("reset")); + else { + const prepared = prepareEnvelope(input, { signal }); + results.push("status" in prepared ? prepared : await processEnvelope(prepared, epoch)); } - return result; - } finally { - if (stagedPayloads.get(envelopeId) === payload) stagedPayloads.delete(envelopeId); } + return Object.freeze(results); + }); + } + + return { + ...telemetry, + ingestEnvelope, + ingestBatch, + ingestPayload, + ingestPayloadBatch(inputs) { + const rejected = disposed ? "disposed" : pendingCount() >= maxPending ? "queue_full" : undefined; + const entries = Array.from(inputs); + if (rejected) return Promise.resolve(Object.freeze(entries.map(() => ignored(rejected)))); + // One admitted queue operation keeps every batch entry ahead of later payload calls. + // Prepare each entry when reached so a large batch cannot allocate unbounded byte copies. + return enqueue(async epoch => { + const results: IngestResult[] = []; + for (const input of entries) { + if (disposed) results.push(ignored("disposed")); + else if (epoch !== generation) results.push(ignored("reset")); + else { + const prepared = preparePayload(input); + results.push( + "status" in prepared + ? prepared + : await processPayload(prepared.input, prepared.body, prepared.contentType, prepared.encoding, epoch), + ); + } + } + return Object.freeze(results); + }); }, - getRawPayload(envelopeId) { + getDecodedPayload(envelopeId) { const payload = rawPayloads.get(envelopeId); return payload && { body: Buffer.from(payload.body), contentType: payload.contentType }; }, + getStats() { + const stats = telemetry.getStats(); + return Object.freeze({ ...stats, pending: pendingCount() }); + }, + async drain() { + // Capture both barriers now: work submitted after this call cannot extend the drain. + const wireBarrier = queue; + const coreBarrier = telemetry.drain(); + await Promise.all([wireBarrier, coreBarrier]); + }, + prune() { + if (disposed) return Promise.resolve(); + if (pruning) return pruning; + const epoch = generation; + // A single control continuation does not consume an ingestion admission slot. + const operation = queue.then(async () => { + if (!disposed && epoch === generation) await telemetry.prune(); + }); + queue = operation.catch(() => {}); + pruning = operation; + const release = () => { + if (pruning === operation) pruning = undefined; + }; + void operation.then(release, release); + return operation; + }, reset() { generation += 1; + pruning = undefined; + queue = Promise.resolve(); + pending.clear(); + coreAdmissions.clear(); + for (const controller of decodeControllers) controller.abort(); rawPayloads.clear(); stagedPayloads.clear(); + rawBytes = 0; telemetry.reset(); }, dispose() { disposed = true; + pruning = undefined; generation += 1; + queue = Promise.resolve(); + pending.clear(); + coreAdmissions.clear(); + for (const controller of decodeControllers) controller.abort(); rawPayloads.clear(); stagedPayloads.clear(); + rawBytes = 0; telemetry.dispose(); }, }; diff --git a/packages/spotlight/src/server/formatters/human/utils.ts b/packages/spotlight/src/server/formatters/human/utils.ts index 5757c9241..b30e957f8 100644 --- a/packages/spotlight/src/server/formatters/human/utils.ts +++ b/packages/spotlight/src/server/formatters/human/utils.ts @@ -1,9 +1,7 @@ -import type { Envelope } from "@sentry/core"; import chalk from "chalk"; -import { parseBrowserFromUserAgent } from "../../routes/stream/userAgent.ts"; -export const SOURCE_TYPES = ["browser", "mobile", "server"] as const; -export type SourceType = (typeof SOURCE_TYPES)[number]; +import { SOURCE_TYPES, type SourceType } from "../../../telemetry/envelope/source.ts"; +export { SOURCE_TYPES, type SourceType, inferEnvelopeSource } from "../../../telemetry/envelope/source.ts"; export const LOG_LEVELS = ["error", "warning", "log", "info", "trace", "debug"] as const; export type LogLevel = (typeof LOG_LEVELS)[number]; @@ -37,154 +35,6 @@ export const LOG_LEVEL_COLORS: Record string> = { debug: chalk.dim, }; -/** - * Helper to detect if a User-Agent string is from a browser - */ -function isBrowserUserAgent(userAgent: string): boolean { - const parsed = parseBrowserFromUserAgent(userAgent); - return ( - parsed !== "unknown" && - (parsed.includes("Chrome") || parsed.includes("Firefox") || parsed.includes("Safari") || parsed.includes("Edge")) - ); -} - -/** - * Check if a platform string indicates a server environment - */ -function isServerPlatform(platform: string | undefined): boolean { - return ( - platform === "node" || - platform === "python" || - platform === "ruby" || - platform === "php" || - platform === "java" || - platform === "go" || - platform === "rust" || - platform === "perl" || - platform === "elixir" || - platform === "csharp" || - platform === "dotnet" - ); -} - -/** - * Try to infer source from a single event's properties - * Returns the source if deterministically detected, null otherwise - */ -function inferSourceFromEvent(event: any): SourceType | null { - if (!event) return null; - - // Runtime tags check - deterministic browser - if (event.tags?.runtime === "browser") { - return "browser"; - } - - // Runtime context indicates server - if (event.contexts?.runtime?.name) { - return "server"; - } - - // server_name is a server-specific field - if (event.server_name) { - return "server"; - } - - // Platform check - if (isServerPlatform(event.platform)) { - return "server"; - } - - return null; -} - -/** - * Infer the source of an envelope as browser, mobile, or server using multiple signals - * Priority order: - * 1. SDK name (mobile detection) - * 2. Sender User-Agent (from HTTP request header) - * 3. Platform & Runtime tags (from event payloads - scans all events) - * 4. SDK name (browser/server fallback) - * - * Rules based on https://release-registry.services.sentry.io/sdks - * - * @param envelopeHeader The envelope header containing SDK info and spotlight extensions - * @param eventsOrEvent Single event or array of events to scan for source signals (scans until deterministic match) - */ -export function inferEnvelopeSource(envelopeHeader: Envelope[0], eventsOrEvent?: any | any[]): SourceType { - // Normalize to array for consistent handling - const events = eventsOrEvent ? (Array.isArray(eventsOrEvent) ? eventsOrEvent : [eventsOrEvent]) : []; - const sdkName = envelopeHeader?.sdk?.name || ""; - - // 1. Mobile check (unchanged - already reliable from SDK name) - // Mobile: Native mobile platforms and frameworks - if ( - sdkName.includes("cocoa") || - sdkName.includes("android") || - sdkName.includes("react-native") || - sdkName.includes("flutter") || - sdkName.includes("capacitor") || - sdkName.includes("cordova") || - sdkName.includes("xamarin") || - sdkName.includes("maui") || - sdkName.includes("unity") || - sdkName.includes("kotlin.kmp") - ) { - return "mobile"; - } - - // 2. Sender User-Agent check - const senderUserAgent = (envelopeHeader as any).__spotlight_sender_user_agent; - if (senderUserAgent && typeof senderUserAgent === "string") { - if (isBrowserUserAgent(senderUserAgent)) { - return "browser"; - } - // Server SDKs send server UAs like "Node.js/*", "Python-urllib/*", etc. - // If we have a non-browser UA, we continue to further checks - } - - // 3. Scan all events for deterministic source signals - for (const event of events) { - const source = inferSourceFromEvent(event); - if (source) { - return source; - } - } - - // 4. SDK name check (existing logic as fallback) - // Browser: JavaScript frameworks/libraries (excluding server/native runtimes and meta-frameworks) - if ( - sdkName.startsWith("sentry.javascript.") && - !sdkName.includes("node") && - !sdkName.includes("bun") && - !sdkName.includes("deno") && - !sdkName.includes("electron") && - !sdkName.includes("serverless") && - !sdkName.includes("cloudflare") && - !sdkName.includes("vercel-edge") && - !sdkName.includes("wasm") && - !sdkName.includes("opentelemetry") && - !sdkName.includes("nextjs") && - !sdkName.includes("remix") && - !sdkName.includes("gatsby") && - !sdkName.includes("astro") && - !sdkName.includes("nuxt") && - !sdkName.includes("sveltekit") && - !sdkName.includes("solidstart") && - !sdkName.includes("nestjs") && - !sdkName.includes("tanstackstart") - ) { - return "browser"; - } - - // Special case: Blazor WebAssembly runs in browser - if (sdkName.includes("blazor.webassembly")) { - return "browser"; - } - - // Server: Everything else (node, python, ruby, go, php, java, dotnet, etc.) - return "server"; -} - /** * Format timestamp as local time HH:MM:SS */ diff --git a/packages/spotlight/src/server/parser/processEnvelope.ts b/packages/spotlight/src/server/parser/processEnvelope.ts index 268cc0b43..297b33743 100644 --- a/packages/spotlight/src/server/parser/processEnvelope.ts +++ b/packages/spotlight/src/server/parser/processEnvelope.ts @@ -1,161 +1,19 @@ -import type { Envelope, EnvelopeItem } from "@sentry/core"; -import { type UUID, uuidv7obj } from "uuidv7"; -import { SUPPORTED_EVENT_TYPES } from "../../telemetry/constants/sentry.ts"; -import { RAW_TYPES } from "../constants.ts"; -import { type SourceType, inferEnvelopeSource } from "../formatters/human/utils.ts"; +import { type DecodedEnvelope, decodeEnvelope } from "../../telemetry/envelope/codec.ts"; import { logger } from "../logger.ts"; import type { RawEventContext } from "./types.ts"; -/** - * Spotlight-specific extensions added to envelope headers for internal tracking - */ -export type SpotlightEnvelopeExtensions = { - __spotlight_envelope_id: UUID; - __spotlight_sender_user_agent?: string; - __spotlight_inferred_source?: SourceType; -}; +export type { SpotlightEnvelopeExtensions } from "../../telemetry/envelope/codec.ts"; +export type ParsedEnvelope = { envelope: DecodedEnvelope; rawEnvelope: RawEventContext }; -export type ParsedEnvelope = { - envelope: [Envelope[0] & SpotlightEnvelopeExtensions, Envelope[1]]; - rawEnvelope: RawEventContext; -}; - -const TEXT_CONTENT_TYPES = new Set([ - "text/plain", - "text/css", - "text/csv", - "text/html", - "text/javascript", - "text/json", - "text/x-json", - "application/json", - "application/ld+json", - "application/json", -]); - -/** - * Implements parser for - * @see https://develop.sentry.dev/sdk/envelopes/#serialization-format - * @param rawEvent Envelope data - * @param senderUserAgent Optional User-Agent header from the HTTP request sending the envelope - * @param reportError Optional error observer; defaults to the legacy sidecar logger - * @param strict Reject truncated payloads and invalid explicit lengths - * @returns parsed envelope - */ +/** Legacy sidecar wrapper: preserve permissive parsing and default logger reporting. */ export function processEnvelope( rawEvent: RawEventContext, senderUserAgent?: string, reportError: (error: unknown) => void = error => logger.error(error), strict = false, ): ParsedEnvelope | null { - let buffer = typeof rawEvent.data === "string" ? Uint8Array.from(rawEvent.data, c => c.charCodeAt(0)) : rawEvent.data; - - function readLine(length?: number) { - const cursor = length ?? getLineEnd(buffer); - const line = buffer.subarray(0, cursor); - buffer = buffer.subarray(cursor + 1); - return line; - } - - const envelopeHeader = parseJSONFromBuffer(readLine(), reportError) as Envelope[0]; - if (!envelopeHeader || typeof envelopeHeader !== "object" || Array.isArray(envelopeHeader)) { - reportError("Malformed envelope header, skipping..."); - return null; - } - - envelopeHeader.__spotlight_envelope_id = uuidv7obj(); - - // Store sender User-Agent for improved SDK categorization - if (senderUserAgent) { - envelopeHeader.__spotlight_sender_user_agent = senderUserAgent; - } - - const items: EnvelopeItem[] = []; - while (buffer.length) { - if (strict && buffer.indexOf(0xa) === -1) { - reportError("Truncated envelope item header, skipping envelope..."); - return null; - } - const itemHeader = parseJSONFromBuffer(readLine(), reportError) as EnvelopeItem[0]; - if (!itemHeader || typeof itemHeader !== "object" || Array.isArray(itemHeader)) { - reportError("Malformed envelope item header, skipping envelope..."); - return null; - } - const payloadLength = itemHeader.length; - if ( - strict && - payloadLength !== undefined && - (!Number.isInteger(payloadLength) || - payloadLength < 0 || - payloadLength > buffer.length || - (payloadLength < buffer.length && buffer[payloadLength] !== 0xa)) - ) { - reportError("Malformed envelope item length, skipping envelope..."); - return null; - } - const itemPayloadRaw = readLine(payloadLength); - - let itemPayload: EnvelopeItem[1]; - try { - if ( - RAW_TYPES.has(itemHeader.type) || - (strict && !SUPPORTED_EVENT_TYPES.has(itemHeader.type) && itemHeader.type !== "span") - ) { - const rawPayload = itemPayloadRaw as Buffer; - if ( - !itemHeader.content_type && - // @ts-expect-error -- statsd is an old and deprecated event type but have envelopes for that - itemHeader.type === "statsd" - ) { - // @ts-expect-error -- same as above - itemHeader.content_type = "text/plain"; - } - itemPayload = { - data: rawPayload.toString(TEXT_CONTENT_TYPES.has(itemHeader.content_type as string) ? "utf-8" : "base64"), - } as EnvelopeItem[1]; - } else { - itemPayload = parseJSONFromBuffer(itemPayloadRaw, reportError); - if (!itemPayload) { - reportError(itemHeader); - } else if (itemHeader.type) { - // data sanitization - // @ts-expect-error ts(2339) -- We should really stop adding type to payloads - itemPayload.type = itemHeader.type; - } - } - } catch (err) { - itemPayload = itemPayloadRaw; - reportError(err); - } - - items.push([itemHeader, itemPayload] as EnvelopeItem); - } - - // Infer the envelope source (browser, server, or mobile) for UI display - // Scan all events to find a deterministic match - const eventPayloads = items.map(([, payload]) => payload); - envelopeHeader.__spotlight_inferred_source = inferEnvelopeSource(envelopeHeader, eventPayloads); - - return { - envelope: [envelopeHeader, items] as ParsedEnvelope["envelope"], - rawEnvelope: rawEvent, - }; -} - -function getLineEnd(data: Uint8Array): number { - let end = data.indexOf(0xa); - if (end === -1) { - end = data.length; - } - - return end; -} - -function parseJSONFromBuffer(data: Uint8Array, reportError: (error: unknown) => void): T { - try { - return JSON.parse(new TextDecoder().decode(data)) as T; - } catch (err) { - reportError(err); - return null as T; - } + const bytes = + typeof rawEvent.data === "string" ? Uint8Array.from(rawEvent.data, c => c.charCodeAt(0)) : rawEvent.data; + const envelope = decodeEnvelope(bytes, senderUserAgent, reportError, strict); + return envelope ? { envelope, rawEnvelope: rawEvent } : null; } diff --git a/packages/spotlight/src/server/routes/stream/userAgent.ts b/packages/spotlight/src/server/routes/stream/userAgent.ts index 7b6fe0e89..634fc68e2 100644 --- a/packages/spotlight/src/server/routes/stream/userAgent.ts +++ b/packages/spotlight/src/server/routes/stream/userAgent.ts @@ -1,57 +1 @@ -/** - * Extracts browser name and major version from a User-Agent string - * Returns format like "Chrome/131" or "Firefox/122" - */ -export function parseBrowserFromUserAgent(userAgent: string): string { - if (!userAgent) return "unknown"; - - // Check for Chrome (but not Edge which also contains Chrome) - if (userAgent.includes("Chrome/") && !userAgent.includes("Edg/")) { - const match = userAgent.match(/Chrome\/([\d]+)/); - return match ? `Chrome/${match[1]}` : "Chrome"; - } - - // Check for Firefox - if (userAgent.includes("Firefox/")) { - const match = userAgent.match(/Firefox\/([\d]+)/); - return match ? `Firefox/${match[1]}` : "Firefox"; - } - - // Check for Safari (but not Chrome which also contains Safari) - if (userAgent.includes("Safari/") && !userAgent.includes("Chrome")) { - const match = userAgent.match(/Version\/([\d]+)/); - return match ? `Safari/${match[1]}` : "Safari"; - } - - // Check for Edge - if (userAgent.includes("Edg/")) { - const match = userAgent.match(/Edg\/([\d]+)/); - return match ? `Edge/${match[1]}` : "Edge"; - } - - // Generic fallback - try to extract first meaningful part - const match = userAgent.match(/^([^\/\s\(]+)/); - return match ? match[1] : "unknown"; -} - -/** - * Builds a client identifier from browser info and optional hostname - * Format: clientName/version (metadata) - * Example: "ui/Chrome/131 (example.com)" - */ -export function buildClientIdentifier(clientName: string, userAgent?: string, hostname?: string): string { - let clientId = clientName; - - if (userAgent) { - const browserInfo = parseBrowserFromUserAgent(userAgent); - if (browserInfo !== "unknown") { - clientId = `${clientName}/${browserInfo}`; - } - } - - if (hostname && hostname !== "localhost") { - clientId += ` (${hostname})`; - } - - return clientId; -} +export { buildClientIdentifier, parseBrowserFromUserAgent } from "../../../telemetry/envelope/userAgent.ts"; diff --git a/packages/spotlight/src/server/sdk.spec.ts b/packages/spotlight/src/server/sdk.spec.ts index 66ae75448..cf45b4652 100644 --- a/packages/spotlight/src/server/sdk.spec.ts +++ b/packages/spotlight/src/server/sdk.spec.ts @@ -35,7 +35,7 @@ function span(overrides: Record = {}) { describe("shared Telemetry SDK", () => { test("ingests compressed v2 spans into traces and query summaries without a browser", async () => { const spotlight = createPipeline(); - const result = await spotlight.ingest({ body: gzipSync(fixture), encoding: "gzip", contentType }); + const result = await spotlight.ingestPayload({ body: gzipSync(fixture), encoding: "gzip", contentType }); if (result.status !== "processed") throw new Error("Expected processed envelope"); expect(spotlight.getQuerySummaries()).toMatchObject([ { @@ -49,7 +49,7 @@ describe("shared Telemetry SDK", () => { expect(spotlight.getTrace(traceId)?.spanTree[0].children?.[0].span_id).toBe(queryId); expect(spotlight.getSpans({ traceId })).toHaveLength(2); expect(spotlight.getEnvelope(result.envelopeId)?.[1][0][1]).toMatchObject({ version: 2, items: [{}, {}] }); - expect(spotlight.getRawPayload(result.envelopeId)?.body).toEqual(fixture); + expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual(fixture); expect(spotlight.getEvents()[0]).toMatchObject({ timestamp: 1793000000250, span: { timestamp: 1793000000250, start_timestamp: 1793000000125 }, @@ -178,14 +178,14 @@ describe("shared Telemetry SDK", () => { test("isolates instance state and resets buffer and processed data together", async () => { const first = createPipeline(); const second = createPipeline(); - const ingested = await first.ingest({ body: fixture, contentType }); + const ingested = await first.ingestPayload({ body: fixture, contentType }); expect(second.getTraces()).toEqual([]); expect(second.getSdks()).toEqual([]); first.reset(); expect(first.getTraces()).toEqual([]); expect(first.getQuerySummaries()).toEqual([]); expect(first.getEnvelopes()).toEqual([]); - if (ingested.status === "processed") expect(first.getRawPayload(ingested.envelopeId)).toBeUndefined(); + if (ingested.status === "processed") expect(first.getDecodedPayload(ingested.envelopeId)).toBeUndefined(); }); test("awaits optional stacktrace enrichment before exposing errors and trace correlation", async () => { @@ -295,7 +295,7 @@ describe("shared Telemetry SDK", () => { const spotlight = createPipeline(); const observed: number[] = []; const unsubscribe = spotlight.subscribe(() => observed.push(spotlight.getQuerySummaries().length)); - await spotlight.ingest({ body: fixture, contentType }); + await spotlight.ingestPayload({ body: fixture, contentType }); expect(observed.at(-1)).toBe(1); unsubscribe(); const count = observed.length; @@ -403,7 +403,7 @@ describe("shared Telemetry SDK", () => { new URL("../../_fixtures/continuous_profiling/profile_chunk_1.txt", import.meta.url), ); for (const body of profileFirst ? [profile, transaction] : [transaction, profile]) { - await spotlight.ingest({ body, contentType }); + await spotlight.ingestPayload({ body, contentType }); } const profileTraceId = "f1e2d3c4b5a6978012345678901234ab"; expect(spotlight.getProfile(profileTraceId)?.samples.length).toBeGreaterThan(0); diff --git a/packages/spotlight/src/telemetry/capabilities.ts b/packages/spotlight/src/telemetry/capabilities.ts new file mode 100644 index 000000000..9df693d2b --- /dev/null +++ b/packages/spotlight/src/telemetry/capabilities.ts @@ -0,0 +1,18 @@ +/** Wire formats understood by the portable SDK. This describes processing, not transport codecs. */ +export const SDK_CAPABILITIES = Object.freeze({ + schemaVersion: 1 as const, + spanFormats: Object.freeze(["legacy", "v2"] as const), + profileFormats: Object.freeze(["v1", "v2"] as const), + itemTypes: Object.freeze([ + "event", + "error", + "transaction", + "span", + "log", + "profile", + "profile_chunk", + "feedback", + "attachment", + ] as const), + timestampUnit: "unix-milliseconds" as const, +}); diff --git a/packages/spotlight/src/telemetry/client.spec.ts b/packages/spotlight/src/telemetry/client.spec.ts index 40d4e52f7..ba37442e5 100644 --- a/packages/spotlight/src/telemetry/client.spec.ts +++ b/packages/spotlight/src/telemetry/client.spec.ts @@ -185,7 +185,7 @@ test("retention preserves identities of profile spans derived from retained enve await client.ingestEnvelope(spans("evict-me")); for (const file of ["transaction.txt", "profile_chunk_1.txt"]) { const body = readFileSync(new URL(`../../_fixtures/continuous_profiling/${file}`, import.meta.url)); - const result = await client.ingest({ body }); + const result = await client.ingestPayload({ body }); expect(result.status).toBe("processed"); } const traceId = "f1e2d3c4b5a6978012345678901234ab"; diff --git a/packages/spotlight/src/telemetry/contracts.ts b/packages/spotlight/src/telemetry/contracts.ts new file mode 100644 index 000000000..a554c35ce --- /dev/null +++ b/packages/spotlight/src/telemetry/contracts.ts @@ -0,0 +1,76 @@ +/** Machine-readable ingestion outcomes. Messages are explanatory, never control flow. */ +export type DiagnosticCode = + | "invalid_envelope" + | "invalid_span" + | "invalid_body" + | "invalid_content_type" + | "unsupported_encoding" + | "decompression_failed" + | "payload_too_large" + | "decoded_payload_too_large" + | "invalid_delivery_id" + | "enrichment_failed" + | "enrichment_timeout" + | "subscriber_failed" + | "unsupported_version"; +export type Diagnostic = { + readonly code: DiagnosticCode; + readonly message: string; + readonly severity: "warning" | "error"; + readonly itemIndex?: number; + readonly spanIndex?: number; +}; +export type IgnoreReason = + | "duplicate" + | "unsupported_items" + | "unsupported_content_type" + | "reset" + | "disposed" + | "aborted" + | "queue_full"; +export type IngestCounts = { + readonly events: number; + readonly spans: number; + readonly logs: number; + readonly skippedItems: number; + readonly duplicateEvents: number; +}; +export type ReceiptMetadata = { + readonly revision: number; + readonly retained: boolean; + readonly counts: IngestCounts; +}; +export type IngestOutcome = + | { readonly status: "processed"; readonly envelopeId: string; readonly diagnostics?: readonly Diagnostic[] } + | { + readonly status: "ignored"; + readonly reason: IgnoreReason; + readonly envelopeId?: string; + readonly diagnostics?: readonly Diagnostic[]; + } + | { readonly status: "invalid"; readonly diagnostics: readonly Diagnostic[] }; +export type IngestResult = ReceiptMetadata & + ( + | { readonly status: "processed"; readonly envelopeId: string; readonly diagnostics: readonly Diagnostic[] } + | { + readonly status: "ignored"; + readonly reason: IgnoreReason; + readonly envelopeId?: string; + readonly diagnostics: readonly Diagnostic[]; + } + | { readonly status: "invalid"; readonly diagnostics: readonly Diagnostic[] } + ); +export type IngestOptions = { deliveryId?: string; signal?: AbortSignal }; +export type BatchOptions = Pick; +export type ChangeMetadata = { + readonly type: "ingest" | "evict" | "reset" | "dispose"; + readonly revision: number; + readonly envelopeIds: readonly string[]; +}; +export type IngestStats = { + readonly pending: number; + readonly retainedEnvelopes: number; + readonly retainedBytes: number; + readonly revision: number; + readonly disposed: boolean; +}; diff --git a/packages/spotlight/src/telemetry/envelope/codec.spec.ts b/packages/spotlight/src/telemetry/envelope/codec.spec.ts new file mode 100644 index 000000000..991911f59 --- /dev/null +++ b/packages/spotlight/src/telemetry/envelope/codec.spec.ts @@ -0,0 +1,62 @@ +import type { Envelope } from "@sentry/core"; +import { describe, expect, test, vi } from "vitest"; +import { normalizeEnvelope } from "../internal/envelopes.ts"; +import { decodeEnvelope } from "./codec.ts"; + +const encode = (value: string) => new TextEncoder().encode(value); + +describe("portable byte envelope codec", () => { + test("parses Uint8Array text attachment bytes with explicit byte lengths", () => { + const text = "é\nhello"; + const payload = encode(text); + const header = encode( + `{}\n${JSON.stringify({ type: "attachment", content_type: "text/plain", length: payload.length })}\n`, + ); + const bytes = new Uint8Array(header.length + payload.length); + bytes.set(header); + bytes.set(payload, header.length); + const envelope = decodeEnvelope(bytes, undefined, undefined, true)!; + expect(envelope[1][0][1]).toEqual({ data: text }); + expect(envelope[0].__spotlight_inferred_source).toBe("server"); + }); + + test("uses the same text and binary policies for parsed and wire envelopes", () => { + for (const content_type of ["text/plain", "application/json", "application/octet-stream", undefined]) { + const payload = encode("\ufeffhéllo"); + const itemHeader = { type: "attachment", content_type, length: payload.length }; + const wireHeader = encode(`{}\n${JSON.stringify(itemHeader)}\n`); + const wire = new Uint8Array(wireHeader.length + payload.length); + wire.set(wireHeader); + wire.set(payload, wireHeader.length); + const decoded = decodeEnvelope(wire, undefined, undefined, true)!; + const parsed = normalizeEnvelope([{}, [[itemHeader, payload]]] as Envelope); + expect(decoded[1]).toEqual(parsed[1]); + } + }); + + test("retains the legacy statsd default text policy", () => { + const wire = decodeEnvelope(encode('{}\n{"type":"statsd"}\ncount:1|c'), undefined, undefined, true)!; + const parsed = normalizeEnvelope([{}, [[{ type: "statsd" }, "count:1|c"]]] as unknown as Envelope); + expect(wire[1]).toEqual(parsed[1]); + expect(wire[1][0][1]).toEqual({ data: "count:1|c" }); + }); + + test("preserves permissive truncation handling while strict decoding rejects it", () => { + const bytes = encode('{}\n{"type":"attachment","length":30}\nshort'); + const errors = vi.fn(); + expect(decodeEnvelope(bytes, undefined, errors)).not.toBeNull(); + expect(errors).not.toHaveBeenCalled(); + expect(decodeEnvelope(bytes, undefined, errors, true)).toBeNull(); + expect(errors).toHaveBeenCalledOnce(); + }); + + test.each([ + ["sentry.javascript.nextjs", "Mozilla/5.0 Chrome/131.0.0.0 Safari/537.36", "browser"], + ["sentry.cocoa", "Mozilla/5.0 Safari/605", "mobile"], + ["sentry.javascript.node", "Node.js/24", "server"], + ])("preserves source inference for %s", (name, userAgent, source) => { + const envelope = decodeEnvelope(encode(`${JSON.stringify({ sdk: { name, version: "1" } })}\n`), userAgent)!; + expect(envelope[0].__spotlight_sender_user_agent).toBe(userAgent); + expect(envelope[0].__spotlight_inferred_source).toBe(source); + }); +}); diff --git a/packages/spotlight/src/telemetry/envelope/codec.ts b/packages/spotlight/src/telemetry/envelope/codec.ts new file mode 100644 index 000000000..8e5aa2215 --- /dev/null +++ b/packages/spotlight/src/telemetry/envelope/codec.ts @@ -0,0 +1,137 @@ +import type { Envelope, EnvelopeItem } from "@sentry/core"; +import { type UUID, uuidv7obj } from "uuidv7"; +import { RAW_TYPES, SUPPORTED_EVENT_TYPES } from "../constants/sentry.ts"; +import { opaquePayload } from "./payload.ts"; +import { type SourceType, inferEnvelopeSource } from "./source.ts"; + +/** + * Spotlight-specific extensions added to envelope headers for internal tracking + */ +export type SpotlightEnvelopeExtensions = { + __spotlight_envelope_id: UUID; + __spotlight_sender_user_agent?: string; + __spotlight_inferred_source?: SourceType; +}; + +export type DecodedEnvelope = [Envelope[0] & SpotlightEnvelopeExtensions, Envelope[1]]; + +/** + * Implements parser for + * @see https://develop.sentry.dev/sdk/envelopes/#serialization-format + * @param data Serialized UTF-8 envelope bytes + * @param senderUserAgent Optional User-Agent header from the HTTP request sending the envelope + * @param reportError Optional error observer; silent by default + * @param strict Reject truncated payloads and invalid explicit lengths + * @returns parsed envelope + */ +export function decodeEnvelope( + data: Uint8Array, + senderUserAgent?: string, + reportError: (error: unknown) => void = () => {}, + strict = false, +): DecodedEnvelope | null { + let buffer = data; + + function readLine(length?: number) { + const cursor = length ?? getLineEnd(buffer); + const line = buffer.subarray(0, cursor); + buffer = buffer.subarray(cursor + 1); + return line; + } + + const envelopeHeader = parseJSONFromBuffer(readLine(), reportError) as Envelope[0]; + if (!envelopeHeader || typeof envelopeHeader !== "object" || Array.isArray(envelopeHeader)) { + reportError("Malformed envelope header, skipping..."); + return null; + } + + envelopeHeader.__spotlight_envelope_id = uuidv7obj(); + + // Store sender User-Agent for improved SDK categorization + if (senderUserAgent) { + envelopeHeader.__spotlight_sender_user_agent = senderUserAgent; + } + + const items: EnvelopeItem[] = []; + while (buffer.length) { + if (strict && buffer.indexOf(0xa) === -1) { + reportError("Truncated envelope item header, skipping envelope..."); + return null; + } + const itemHeader = parseJSONFromBuffer(readLine(), reportError) as EnvelopeItem[0]; + if (!itemHeader || typeof itemHeader !== "object" || Array.isArray(itemHeader)) { + reportError("Malformed envelope item header, skipping envelope..."); + return null; + } + const payloadLength = itemHeader.length; + if ( + strict && + payloadLength !== undefined && + (!Number.isInteger(payloadLength) || + payloadLength < 0 || + payloadLength > buffer.length || + (payloadLength < buffer.length && buffer[payloadLength] !== 0xa)) + ) { + reportError("Malformed envelope item length, skipping envelope..."); + return null; + } + const itemPayloadRaw = readLine(payloadLength); + + let itemPayload: EnvelopeItem[1]; + try { + if ( + RAW_TYPES.has(itemHeader.type) || + (strict && !SUPPORTED_EVENT_TYPES.has(itemHeader.type) && itemHeader.type !== "span") + ) { + if ( + !itemHeader.content_type && + // @ts-expect-error -- statsd is an old and deprecated event type but have envelopes for that + itemHeader.type === "statsd" + ) { + // @ts-expect-error -- same as above + itemHeader.content_type = "text/plain"; + } + itemPayload = opaquePayload(itemPayloadRaw, itemHeader.content_type as string) as EnvelopeItem[1]; + } else { + itemPayload = parseJSONFromBuffer(itemPayloadRaw, reportError); + if (!itemPayload) { + reportError(itemHeader); + } else if (itemHeader.type) { + // data sanitization + // @ts-expect-error ts(2339) -- We should really stop adding type to payloads + itemPayload.type = itemHeader.type; + } + } + } catch (err) { + itemPayload = itemPayloadRaw; + reportError(err); + } + + items.push([itemHeader, itemPayload] as EnvelopeItem); + } + + // Infer the envelope source (browser, server, or mobile) for UI display + // Scan all events to find a deterministic match + const eventPayloads = items.map(([, payload]) => payload); + envelopeHeader.__spotlight_inferred_source = inferEnvelopeSource(envelopeHeader, eventPayloads); + + return [envelopeHeader, items] as DecodedEnvelope; +} + +function getLineEnd(data: Uint8Array): number { + let end = data.indexOf(0xa); + if (end === -1) { + end = data.length; + } + + return end; +} + +function parseJSONFromBuffer(data: Uint8Array, reportError: (error: unknown) => void): T { + try { + return JSON.parse(new TextDecoder().decode(data)) as T; + } catch (err) { + reportError(err); + return null as T; + } +} diff --git a/packages/spotlight/src/telemetry/envelope/payload.ts b/packages/spotlight/src/telemetry/envelope/payload.ts new file mode 100644 index 000000000..d9f4ef896 --- /dev/null +++ b/packages/spotlight/src/telemetry/envelope/payload.ts @@ -0,0 +1,24 @@ +/** The same text/binary representation is used for parsed and serialized opaque items. */ +const TEXT_CONTENT_TYPES = new Set([ + "text/plain", + "text/css", + "text/csv", + "text/html", + "text/javascript", + "text/json", + "text/x-json", + "application/json", + "application/ld+json", +]); + +export function opaquePayload(payload: string | Uint8Array, contentType?: string): { data: string } { + const bytes = typeof payload === "string" ? new TextEncoder().encode(payload) : payload; + if (TEXT_CONTENT_TYPES.has(contentType ?? "")) { + return { data: new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes) }; + } + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return { data: btoa(binary) }; +} diff --git a/packages/spotlight/src/telemetry/envelope/source.ts b/packages/spotlight/src/telemetry/envelope/source.ts new file mode 100644 index 000000000..62cd60239 --- /dev/null +++ b/packages/spotlight/src/telemetry/envelope/source.ts @@ -0,0 +1,153 @@ +import type { Envelope } from "@sentry/core"; +import { parseBrowserFromUserAgent } from "./userAgent.ts"; + +export const SOURCE_TYPES = ["browser", "mobile", "server"] as const; +export type SourceType = (typeof SOURCE_TYPES)[number]; + +/** + * Helper to detect if a User-Agent string is from a browser + */ +function isBrowserUserAgent(userAgent: string): boolean { + const parsed = parseBrowserFromUserAgent(userAgent); + return ( + parsed !== "unknown" && + (parsed.includes("Chrome") || parsed.includes("Firefox") || parsed.includes("Safari") || parsed.includes("Edge")) + ); +} + +/** + * Check if a platform string indicates a server environment + */ +function isServerPlatform(platform: string | undefined): boolean { + return ( + platform === "node" || + platform === "python" || + platform === "ruby" || + platform === "php" || + platform === "java" || + platform === "go" || + platform === "rust" || + platform === "perl" || + platform === "elixir" || + platform === "csharp" || + platform === "dotnet" + ); +} + +/** + * Try to infer source from a single event's properties + * Returns the source if deterministically detected, null otherwise + */ +function inferSourceFromEvent(event: any): SourceType | null { + if (!event) return null; + + // Runtime tags check - deterministic browser + if (event.tags?.runtime === "browser") { + return "browser"; + } + + // Runtime context indicates server + if (event.contexts?.runtime?.name) { + return "server"; + } + + // server_name is a server-specific field + if (event.server_name) { + return "server"; + } + + // Platform check + if (isServerPlatform(event.platform)) { + return "server"; + } + + return null; +} + +/** + * Infer the source of an envelope as browser, mobile, or server using multiple signals + * Priority order: + * 1. SDK name (mobile detection) + * 2. Sender User-Agent (from HTTP request header) + * 3. Platform & Runtime tags (from event payloads - scans all events) + * 4. SDK name (browser/server fallback) + * + * Rules based on https://release-registry.services.sentry.io/sdks + * + * @param envelopeHeader The envelope header containing SDK info and spotlight extensions + * @param eventsOrEvent Single event or array of events to scan for source signals (scans until deterministic match) + */ +export function inferEnvelopeSource(envelopeHeader: Envelope[0], eventsOrEvent?: any | any[]): SourceType { + // Normalize to array for consistent handling + const events = eventsOrEvent ? (Array.isArray(eventsOrEvent) ? eventsOrEvent : [eventsOrEvent]) : []; + const sdkName = envelopeHeader?.sdk?.name || ""; + + // 1. Mobile check (unchanged - already reliable from SDK name) + // Mobile: Native mobile platforms and frameworks + if ( + sdkName.includes("cocoa") || + sdkName.includes("android") || + sdkName.includes("react-native") || + sdkName.includes("flutter") || + sdkName.includes("capacitor") || + sdkName.includes("cordova") || + sdkName.includes("xamarin") || + sdkName.includes("maui") || + sdkName.includes("unity") || + sdkName.includes("kotlin.kmp") + ) { + return "mobile"; + } + + // 2. Sender User-Agent check + const senderUserAgent = (envelopeHeader as any).__spotlight_sender_user_agent; + if (senderUserAgent && typeof senderUserAgent === "string") { + if (isBrowserUserAgent(senderUserAgent)) { + return "browser"; + } + // Server SDKs send server UAs like "Node.js/*", "Python-urllib/*", etc. + // If we have a non-browser UA, we continue to further checks + } + + // 3. Scan all events for deterministic source signals + for (const event of events) { + const source = inferSourceFromEvent(event); + if (source) { + return source; + } + } + + // 4. SDK name check (existing logic as fallback) + // Browser: JavaScript frameworks/libraries (excluding server/native runtimes and meta-frameworks) + if ( + sdkName.startsWith("sentry.javascript.") && + !sdkName.includes("node") && + !sdkName.includes("bun") && + !sdkName.includes("deno") && + !sdkName.includes("electron") && + !sdkName.includes("serverless") && + !sdkName.includes("cloudflare") && + !sdkName.includes("vercel-edge") && + !sdkName.includes("wasm") && + !sdkName.includes("opentelemetry") && + !sdkName.includes("nextjs") && + !sdkName.includes("remix") && + !sdkName.includes("gatsby") && + !sdkName.includes("astro") && + !sdkName.includes("nuxt") && + !sdkName.includes("sveltekit") && + !sdkName.includes("solidstart") && + !sdkName.includes("nestjs") && + !sdkName.includes("tanstackstart") + ) { + return "browser"; + } + + // Special case: Blazor WebAssembly runs in browser + if (sdkName.includes("blazor.webassembly")) { + return "browser"; + } + + // Server: Everything else (node, python, ruby, go, php, java, dotnet, etc.) + return "server"; +} diff --git a/packages/spotlight/src/telemetry/envelope/userAgent.ts b/packages/spotlight/src/telemetry/envelope/userAgent.ts new file mode 100644 index 000000000..7b6fe0e89 --- /dev/null +++ b/packages/spotlight/src/telemetry/envelope/userAgent.ts @@ -0,0 +1,57 @@ +/** + * Extracts browser name and major version from a User-Agent string + * Returns format like "Chrome/131" or "Firefox/122" + */ +export function parseBrowserFromUserAgent(userAgent: string): string { + if (!userAgent) return "unknown"; + + // Check for Chrome (but not Edge which also contains Chrome) + if (userAgent.includes("Chrome/") && !userAgent.includes("Edg/")) { + const match = userAgent.match(/Chrome\/([\d]+)/); + return match ? `Chrome/${match[1]}` : "Chrome"; + } + + // Check for Firefox + if (userAgent.includes("Firefox/")) { + const match = userAgent.match(/Firefox\/([\d]+)/); + return match ? `Firefox/${match[1]}` : "Firefox"; + } + + // Check for Safari (but not Chrome which also contains Safari) + if (userAgent.includes("Safari/") && !userAgent.includes("Chrome")) { + const match = userAgent.match(/Version\/([\d]+)/); + return match ? `Safari/${match[1]}` : "Safari"; + } + + // Check for Edge + if (userAgent.includes("Edg/")) { + const match = userAgent.match(/Edg\/([\d]+)/); + return match ? `Edge/${match[1]}` : "Edge"; + } + + // Generic fallback - try to extract first meaningful part + const match = userAgent.match(/^([^\/\s\(]+)/); + return match ? match[1] : "unknown"; +} + +/** + * Builds a client identifier from browser info and optional hostname + * Format: clientName/version (metadata) + * Example: "ui/Chrome/131 (example.com)" + */ +export function buildClientIdentifier(clientName: string, userAgent?: string, hostname?: string): string { + let clientId = clientName; + + if (userAgent) { + const browserInfo = parseBrowserFromUserAgent(userAgent); + if (browserInfo !== "unknown") { + clientId = `${clientName}/${browserInfo}`; + } + } + + if (hostname && hostname !== "localhost") { + clientId += ` (${hostname})`; + } + + return clientId; +} diff --git a/packages/spotlight/src/telemetry/index.ts b/packages/spotlight/src/telemetry/index.ts index 4da0d70be..b24e2efe0 100644 --- a/packages/spotlight/src/telemetry/index.ts +++ b/packages/spotlight/src/telemetry/index.ts @@ -1,123 +1,204 @@ import type { Envelope } from "@sentry/core"; -import { createEngine } from "./internal/engine.ts"; -import { normalizeEnvelope, supportsEnvelope, validateEnvelope } from "./internal/envelopes.ts"; -import { readonlyCopy } from "./internal/readonly.ts"; +import { SUPPORTED_EVENT_TYPES } from "./constants/sentry.ts"; import type { + ChangeMetadata, Diagnostic, - EventFilter, + DiagnosticCode, + IgnoreReason, + IngestCounts, + IngestOptions, IngestResult, - Ingester, - IngesterOptions, - IngesterSnapshot, - TraceFilter, -} from "./public.ts"; -import { - getAITraces, - getProfileFunctionSummaries, - getQuerySummaries, - getResourceSummaries, - getSpans, - getWebVitals, -} from "./selectors/index.ts"; +} from "./contracts.ts"; +import { createEngine } from "./internal/engine.ts"; +import { normalizeEnvelope, supportsEnvelope, validateEnvelope } from "./internal/envelopes.ts"; +import { readonlyCopy } from "./internal/readonly.ts"; +import { diagnostic as makeDiagnostic, receipt, reportDiagnostic } from "./internal/results.ts"; +import type { Ingester, IngesterOptions, IngesterSnapshot } from "./public.ts"; +import { createTelemetryReader } from "./reader.ts"; import { normalizeSpanItem } from "./store/utils/spanProcessor.ts"; -import type { SentryErrorEvent, SentryEvent, Trace } from "./types.ts"; -import { isErrorEvent } from "./utils/sentry.ts"; +import type { SentryErrorEvent } from "./types.ts"; import { generateUuidv4 } from "./uuid.ts"; -type RetainedEnvelope = { id: string; envelope: Envelope; receivedAtMs: number }; +type RetainedEnvelope = { id: string; envelope: Envelope; receivedAtMs: number; bytes: number }; +const MiB = 1024 * 1024; +function positive(value: number, name: string) { + if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`${name} must be a positive safe integer`); + return value; +} -/** Portable, independently owned telemetry with immutable committed reads. */ +/** Portable processing with bounded work, immutable committed reads and explicit receipts. */ export function createIngester(options: IngesterOptions = {}): Ingester { - const maxEnvelopes = options.retention?.maxEnvelopes ?? 500; + const maxEnvelopes = positive(options.retention?.maxEnvelopes ?? 500, "retention.maxEnvelopes"); + const maxBytes = positive(options.retention?.maxBytes ?? 64 * MiB, "retention.maxBytes"); + const maxPending = positive(options.maxPending ?? 64, "maxPending"); + const maxEnvelopeBytes = positive(options.maxEnvelopeBytes ?? 32 * MiB, "maxEnvelopeBytes"); + const enrichmentTimeoutMs = positive(options.enrichmentTimeoutMs ?? 5000, "enrichmentTimeoutMs"); const maxAgeMs = options.retention?.maxAgeMs; - if (!Number.isSafeInteger(maxEnvelopes) || maxEnvelopes < 1) - throw new RangeError("retention.maxEnvelopes must be a positive safe integer"); if (maxAgeMs !== undefined && (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0)) throw new RangeError("retention.maxAgeMs must be a positive finite number"); let generation = 0; let disposed = false; let revision = 0; + let pending = 0; let queue: Promise = Promise.resolve(); + let pruning: Promise | undefined; let journal: RetainedEnvelope[] = []; let timer: ReturnType | undefined; + let lifetime = new AbortController(); const identities = new WeakMap(); - const listeners = new Set<(snapshot: IngesterSnapshot) => void>(); + const listeners = new Set<(snapshot: IngesterSnapshot, change: ChangeMetadata) => void>(); let enrichmentCache = new Map(); let currentDiagnostics: Diagnostic[] = []; - function diagnostic(item: Diagnostic) { + const stableCopies = new WeakMap(); + function active(epoch: number) { + return !disposed && epoch === generation; + } + function diagnostic( + code: DiagnosticCode, + message: string, + details: Partial> = {}, + ) { + const item = makeDiagnostic(code, message, details); currentDiagnostics.push(item); - try { - options.onDiagnostic?.(item); - } catch { - /* Diagnostic observers cannot break capture. */ - } + reportDiagnostic(options.onDiagnostic, item); + return item; + } + function invalid(code: DiagnosticCode, message: string, diagnostics?: readonly Diagnostic[]): IngestResult { + const items = diagnostics ?? [makeDiagnostic(code, message)]; + for (const item of items) reportDiagnostic(options.onDiagnostic, item); + return receipt({ status: "invalid", diagnostics: items }, { revision }); + } + function ignored(reason: IgnoreReason, envelopeId?: string, retained = false): IngestResult { + return receipt( + { status: "ignored", reason, ...(envelopeId !== undefined ? { envelopeId } : {}) }, + { revision, retained }, + ); } function makeEngine() { const cache = enrichmentCache; const epoch = generation; + const lifetimeSignal = lifetime.signal; let currentEnvelopeId = ""; + let currentSignal: AbortSignal | undefined; const created = createEngine({ enrichStacktrace: async event => { const envelopeId = currentEnvelopeId; + const inputSignal = currentSignal; const cached = cache.get(event.event_id); if (cached?.envelopeId === envelopeId) { event.exception = structuredClone(cached.exception); return; } - try { - await options.enrichStacktrace?.(event); - } catch { - if (active(epoch) && created.runtime.generation === 0) - diagnostic({ - code: "enrichment_failed", - message: "Stacktrace enrichment failed; captured telemetry was retained.", + if (options.enrichStacktrace) { + // A user hook receives an isolated copy. After timeout/abort, late writes cannot escape. + const enriched = structuredClone(event); + const controller = new AbortController(); + let timeout: ReturnType | undefined; + let completed = false; + let abortListener!: () => void; + let lifetimeListener!: () => void; + try { + await new Promise(resolve => { + const finish = () => { + if (!completed) { + completed = true; + resolve(); + } + }; + const cancel = () => { + controller.abort(); + finish(); + }; + abortListener = cancel; + lifetimeListener = cancel; + inputSignal?.addEventListener("abort", abortListener, { once: true }); + lifetimeSignal.addEventListener("abort", lifetimeListener, { once: true }); + if (inputSignal?.aborted || lifetimeSignal.aborted) { + cancel(); + return; + } + const deadline = Date.now() + enrichmentTimeoutMs; + const checkDeadline = () => { + if (completed) return; + const remaining = deadline - Date.now(); + if (remaining > 0) { + timeout = setTimeout(checkDeadline, Math.min(2147483647, remaining)); + return; + } + if (active(epoch)) + diagnostic( + "enrichment_timeout", + "Stacktrace enrichment exceeded its deadline; telemetry was retained.", + { severity: "warning" }, + ); + cancel(); + }; + checkDeadline(); + void Promise.resolve() + .then(() => options.enrichStacktrace!(enriched, { signal: controller.signal })) + .then( + () => { + if (!completed && active(epoch) && created.runtime.generation === 0 && !controller.signal.aborted) + event.exception = structuredClone(enriched.exception); + finish(); + }, + () => { + if (!completed && active(epoch)) + diagnostic("enrichment_failed", "Stacktrace enrichment failed; telemetry was retained.", { + severity: "warning", + }); + finish(); + }, + ); }); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + inputSignal?.removeEventListener("abort", abortListener); + lifetimeSignal.removeEventListener("abort", lifetimeListener); + } } - if (active(epoch) && created.runtime.generation === 0) + if (active(epoch) && created.runtime.generation === 0 && !inputSignal?.aborted) cache.set(event.event_id, { envelopeId, exception: structuredClone(event.exception) }); }, }); return { ...created, - ingestEnvelope(envelope: Envelope, receivedAtMs: number) { + ingestEnvelope(envelope: Envelope, receivedAtMs: number, signal?: AbortSignal) { currentEnvelopeId = String(envelope[0].__spotlight_envelope_id); + currentSignal = signal; created.runtime.receivedAtMs = receivedAtMs; return created.store.getState().ingestEnvelope(envelope); }, }; } - function invalid(code: string, message: string): IngestResult { - const item = { code, message }; - try { - options.onDiagnostic?.(item); - } catch { - /* Diagnostic observers do not break ingestion. */ - } - return readonlyCopy({ status: "invalid" as const, diagnostics: [item] }); - } let engine = makeEngine(); + function stable(value: T): T { + if (!stableCopies.has(value)) stableCopies.set(value, readonlyCopy(value)); + return stableCopies.get(value) as T; + } function stateSnapshot(): IngesterSnapshot { const state = engine.store.getState(); - return readonlyCopy({ + return Object.freeze({ revision, - envelopes: state.getEnvelopes(), - events: state.getEvents(), - traces: state.getTraces(), - logs: state.getLogs(), - sdks: state.getSdks(), - profiles: state.profilesByTraceId, + envelopes: Object.freeze(state.getEnvelopes().map(stable)), + events: Object.freeze(state.getEvents().map(stable)), + traces: readonlyCopy(state.getTraces()), + logs: Object.freeze(state.getLogs().map(stable)), + sdks: readonlyCopy(state.getSdks()), + profiles: readonlyCopy(state.profilesByTraceId), }); } let snapshot = stateSnapshot(); - function publish() { + function publish(type: ChangeMetadata["type"], envelopeIds: readonly string[] = []) { revision++; snapshot = stateSnapshot(); const committed = snapshot; + const change = Object.freeze({ type, revision, envelopeIds: Object.freeze([...envelopeIds]) }); for (const listener of listeners) { try { - listener(committed); + listener(committed, change); } catch { - diagnostic({ code: "subscriber_failed", message: "A telemetry subscription callback failed." }); + diagnostic("subscriber_failed", "A telemetry subscription callback failed.", { severity: "warning" }); } if (snapshot !== committed) break; } @@ -129,7 +210,8 @@ export function createIngester(options: IngesterOptions = {}): Ingester { function scheduleExpiry() { clearTimer(); if (disposed || maxAgeMs === undefined || !journal.length) return; - const delay = Math.max(1, Math.min(2147483647, journal[0].receivedAtMs + maxAgeMs - Date.now())); + const earliest = Math.min(...journal.map(record => record.receivedAtMs)); + const delay = Math.max(1, Math.min(2147483647, earliest + maxAgeMs - Date.now())); timer = setTimeout(() => { timer = undefined; void client.prune(); @@ -138,203 +220,303 @@ export function createIngester(options: IngesterOptions = {}): Ingester { } function enqueue(task: (epoch: number) => Promise): Promise { const epoch = generation; - const next = queue.then(() => task(epoch)); + pending++; + const next = queue + .then(() => task(epoch)) + .finally(() => { + if (epoch === generation) pending--; + }); queue = next.catch(() => {}); return next; } - function ignored(reason: string, envelopeId?: string): IngestResult { - return readonlyCopy({ status: "ignored" as const, reason, ...(envelopeId ? { envelopeId } : {}), diagnostics: [] }); - } - function active(epoch: number) { - return !disposed && epoch === generation; - } - async function rebuild(retained: RetainedEnvelope[], epoch: number) { + async function rebuild(retained: RetainedEnvelope[], epoch: number, signal?: AbortSignal) { const replacement = makeEngine(); for (const record of retained) { - await replacement.ingestEnvelope(record.envelope, record.receivedAtMs); - if (!active(epoch)) return; + if (signal?.aborted) return false; + await replacement.ingestEnvelope(record.envelope, record.receivedAtMs, signal); + if (!active(epoch) || signal?.aborted) return false; } engine = replacement; const eventIds = engine.store.getState().eventsById; for (const key of enrichmentCache.keys()) if (!eventIds.has(key)) enrichmentCache.delete(key); + return true; } - async function enforceRetention(epoch: number): Promise { + async function enforceRetention(epoch: number, signal?: AbortSignal): Promise { const cutoff = maxAgeMs === undefined ? Number.NEGATIVE_INFINITY : Date.now() - maxAgeMs; - const retained = journal.filter(record => record.receivedAtMs > cutoff).slice(-maxEnvelopes); - if (retained.length === journal.length) return false; - await rebuild(retained, epoch); - if (!active(epoch)) return false; + let retained = journal.filter(record => record.receivedAtMs > cutoff).slice(-maxEnvelopes); + let bytes = retained.reduce((sum, record) => sum + record.bytes, 0); + let trim = 0; + while (bytes > maxBytes && trim < retained.length) bytes -= retained[trim++].bytes; + retained = retained.slice(trim); + if (retained.length === journal.length) return []; + const keep = new Set(retained.map(record => record.id)); + const evicted = journal.filter(record => !keep.has(record.id)).map(record => record.id); + if (!(await rebuild(retained, epoch, signal))) return []; + if (!active(epoch)) return []; journal = retained; - return true; + return evicted; } - function filterEvents(filter?: EventFilter) { - return snapshot.events.filter( - event => - (filter?.traceId === undefined || event.contexts?.trace?.trace_id === filter.traceId) && - (filter?.spanId === undefined || event.contexts?.trace?.span_id === filter.spanId), - ); - } - function filteredTraces(filter?: TraceFilter) { - return snapshot.traces.filter(trace => filter?.traceId === undefined || trace.trace_id === filter.traceId); + function spanKeys() { + const keys = new Set(); + for (const trace of engine.store.getState().tracesById.values()) + for (const id of trace.spans.keys()) keys.add(`${trace.trace_id}:${id}`); + return keys; } - const client: Ingester = { - ingestEnvelope: input => { - if (disposed) return Promise.resolve(ignored("disposed")); - if (!validateEnvelope(input)) - return Promise.resolve( - invalid("invalid_envelope", "Expected a parsed envelope [headers, [[itemHeaders, payload], ...]]."), + const reader = createTelemetryReader(() => snapshot); + function ingestOne(input: unknown, inputOptions: IngestOptions = {}, withinEpoch?: number): Promise { + const ingestOptions = { deliveryId: inputOptions.deliveryId, signal: inputOptions.signal }; + if (disposed) return Promise.resolve(ignored("disposed")); + if (ingestOptions.signal?.aborted) return Promise.resolve(ignored("aborted")); + if (withinEpoch === undefined && pending >= maxPending) return Promise.resolve(ignored("queue_full")); + if ( + ingestOptions.deliveryId !== undefined && + (typeof ingestOptions.deliveryId !== "string" || !ingestOptions.deliveryId.trim()) + ) + return Promise.resolve(invalid("invalid_delivery_id", "deliveryId must be a non-empty string.")); + if (!validateEnvelope(input)) + return Promise.resolve( + invalid("invalid_envelope", "Expected a parsed envelope [headers, [[itemHeaders, payload], ...]]."), + ); + const supported = supportsEnvelope(input); + const inputDiagnostics: Diagnostic[] = []; + let validSpans = 0; + let invalidSpans = 0; + let otherItems = false; + let candidates = 0; + let skippedItems = 0; + for (const [itemIndex, [header, payload]] of input[1].entries()) { + if (header.type !== "span") { + otherItems = true; + if (SUPPORTED_EVENT_TYPES.has(header.type)) candidates++; + else skippedItems++; + continue; + } + const data = payload as Record; + if ("items" in data && data.version !== 2) { + skippedItems++; + inputDiagnostics.push( + makeDiagnostic("unsupported_version", "Unsupported span container version was retained without processing.", { + severity: "warning", + itemIndex, + }), ); - const supported = supportsEnvelope(input); - const inputDiagnostics: Diagnostic[] = []; - let validSpans = 0; - let invalidSpans = 0; - let otherItems = false; - for (const [header, payload] of input[1]) { - if (header.type !== "span") { - otherItems = true; - continue; + continue; + } + const count = Array.isArray(data.items) ? data.items.length : 1; + const valid = normalizeSpanItem(data, { name: "unknown", version: "unknown", lastSeen: 0 }).length; + validSpans += valid; + candidates += valid; + invalidSpans += count - valid; + if (count > valid) + inputDiagnostics.push( + makeDiagnostic("invalid_span", "Malformed span items were skipped; valid telemetry was retained.", { + severity: "warning", + itemIndex, + }), + ); + } + if (invalidSpans > 0 && validSpans === 0 && !otherItems) + return Promise.resolve( + invalid( + "invalid_span", + "Span items are missing valid IDs or timestamps.", + inputDiagnostics.map(item => + makeDiagnostic(item.code, "Span items are missing valid IDs or timestamps.", { + severity: "error", + itemIndex: item.itemIndex, + }), + ), + ), + ); + skippedItems += invalidSpans; + let id: string; + let envelope: Envelope; + let bytes: number; + try { + id = + ingestOptions.deliveryId ?? + String(input[0].__spotlight_envelope_id ?? identities.get(input) ?? generateUuidv4()); + identities.set(input, id); + envelope = normalizeEnvelope(input); + envelope[0].__spotlight_envelope_id = id; + bytes = new TextEncoder().encode(JSON.stringify(envelope)).byteLength; + } catch { + return Promise.resolve(invalid("invalid_envelope", "Envelope data must be cloneable wire data.")); + } + if (bytes > maxEnvelopeBytes) + return Promise.resolve(invalid("payload_too_large", "Parsed envelope exceeds maxEnvelopeBytes.")); + const receivedAtMs = Date.now(); + const process = async (epoch: number): Promise => { + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (ingestOptions.signal?.aborted) return ignored("aborted", id); + currentDiagnostics = []; + for (const item of inputDiagnostics) { + currentDiagnostics.push(item); + reportDiagnostic(options.onDiagnostic, item); + } + const evicted = await enforceRetention(epoch, ingestOptions.signal); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (ingestOptions.signal?.aborted) { + if (evicted.length) publish("evict", evicted); + scheduleExpiry(); + return ignored("aborted", id); + } + const retainedBefore = journal; + const cacheBefore = new Map(enrichmentCache); + async function rollback() { + working.store.getState().resetData(); + if (engine !== working) engine.store.getState().resetData(); + journal = retainedBefore; + enrichmentCache = new Map(cacheBefore); + await rebuild(journal, epoch); + if (active(epoch)) { + if (evicted.length) publish("evict", evicted); + scheduleExpiry(); } - const data = payload as Record; - if ("items" in data && data.version !== 2) continue; - const count = Array.isArray(data.items) ? data.items.length : 1; - const valid = normalizeSpanItem(data, { name: "unknown", version: "unknown", lastSeen: 0 }).length; - validSpans += valid; - invalidSpans += count - valid; } - if (invalidSpans > 0) { - if (validSpans === 0 && !otherItems) - return Promise.resolve(invalid("invalid_span", "Span items are missing valid IDs or timestamps.")); - inputDiagnostics.push({ - code: "invalid_span", - message: "Malformed span items were skipped; valid telemetry was retained.", - }); + if (journal.some(record => record.id === id)) { + if (evicted.length) publish("evict", evicted); + scheduleExpiry(); + return ignored("duplicate", id, true); } - let id: string; - let envelope: Envelope; + const working = engine; + const beforeEvents = new Set(working.store.getState().eventsById.keys()); + const beforeLogs = new Set(working.store.getState().logsById.keys()); + const beforeSpans = spanKeys(); + let failed = false; try { - id = String(input[0].__spotlight_envelope_id ?? identities.get(input) ?? generateUuidv4()); - identities.set(input, id); - envelope = normalizeEnvelope(input); - envelope[0].__spotlight_envelope_id = id; + await working.ingestEnvelope(envelope, receivedAtMs, ingestOptions.signal); } catch { - return Promise.resolve(invalid("invalid_envelope", "Envelope data must be cloneable wire data.")); + failed = true; } - const receivedAtMs = Date.now(); - return enqueue(async epoch => { + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (failed || ingestOptions.signal?.aborted) { + await rollback(); if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); - currentDiagnostics = []; - for (const item of inputDiagnostics) diagnostic(item); - const evicted = await enforceRetention(epoch); - if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); - if (journal.some(record => record.id === id)) { - if (evicted) publish(); - scheduleExpiry(); - return ignored("duplicate", id); - } - const working = engine; - try { - await working.ingestEnvelope(envelope, receivedAtMs); - } catch { - working.store.getState().resetData(); - if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); - await rebuild(journal, epoch); - if (active(epoch)) { - if (evicted) publish(); - scheduleExpiry(); - } - return invalid("invalid_envelope", "Envelope telemetry could not be processed."); + return ingestOptions.signal?.aborted + ? ignored("aborted", id) + : invalid("invalid_envelope", "Envelope telemetry could not be processed."); + } + const state = working.store.getState(); + const addedEvents = Array.from(state.eventsById.keys()).filter(key => !beforeEvents.has(key)).length; + const counts: IngestCounts = { + events: addedEvents, + logs: Array.from(state.logsById.keys()).filter(key => !beforeLogs.has(key)).length, + spans: Array.from(spanKeys()).filter(key => !beforeSpans.has(key)).length, + skippedItems, + duplicateEvents: Math.max(0, candidates - addedEvents), + }; + journal = [...journal, { id, envelope, receivedAtMs, bytes }]; + const afterEvicted = await enforceRetention(epoch, ingestOptions.signal); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + if (ingestOptions.signal?.aborted) { + await rollback(); + return ignored(active(epoch) ? "aborted" : disposed ? "disposed" : "reset", id); + } + publish("ingest", [id, ...evicted, ...afterEvicted]); + if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); + scheduleExpiry(); + return receipt( + supported + ? { status: "processed", envelopeId: id, diagnostics: currentDiagnostics } + : { status: "ignored", reason: "unsupported_items", envelopeId: id, diagnostics: currentDiagnostics }, + { revision, retained: journal.some(record => record.id === id), counts }, + ); + }; + return withinEpoch === undefined ? enqueue(process) : process(withinEpoch); + } + const client: Ingester = { + ...reader, + ingestEnvelope: (input, inputOptions) => ingestOne(input, inputOptions), + ingestBatch(inputs, batchOptions = {}) { + const signal = batchOptions.signal; + if (disposed || pending >= maxPending) { + return Promise.resolve(Object.freeze(Array.from(inputs, () => ignored(disposed ? "disposed" : "queue_full")))); + } + return enqueue(async epoch => { + const results: IngestResult[] = []; + for (const input of inputs) { + results.push( + active(epoch) ? await ingestOne(input, { signal }, epoch) : ignored(disposed ? "disposed" : "reset"), + ); } - if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); - journal.push({ id, envelope, receivedAtMs }); - await enforceRetention(epoch); - if (!active(epoch)) return ignored(disposed ? "disposed" : "reset", id); - publish(); - scheduleExpiry(); - return supported - ? readonlyCopy({ status: "processed" as const, envelopeId: id, diagnostics: currentDiagnostics }) - : readonlyCopy({ - status: "ignored" as const, - reason: "unsupported_items", - envelopeId: id, - diagnostics: currentDiagnostics, - }); + return Object.freeze(results); }); }, - getSnapshot: () => snapshot, - getEnvelope: id => snapshot.envelopes.find(envelope => String(envelope[0].__spotlight_envelope_id) === id), - getEnvelopes: () => snapshot.envelopes, - getEvent: id => snapshot.events.find(event => event.event_id === id), - getEvents: filter => (filter ? Object.freeze(filterEvents(filter)) : snapshot.events), - getErrors: filter => - Object.freeze(filterEvents(filter).filter(event => isErrorEvent(event as SentryEvent))) as ReturnType< - Ingester["getErrors"] - >, - getTrace: id => snapshot.traces.find(trace => trace.trace_id === id), - getTraces: filter => (filter ? Object.freeze(filteredTraces(filter)) : snapshot.traces), - getSpans: filter => Object.freeze(getSpans(filteredTraces(filter) as Trace[])) as ReturnType, - getLog: id => snapshot.logs.find(log => log.id === id), - getLogs: filter => - filter?.traceId === undefined - ? snapshot.logs - : Object.freeze(snapshot.logs.filter(log => log.trace_id === filter.traceId)), - getProfile: id => snapshot.profiles.get(id), - getProfileFunctionSummaries: filter => - readonlyCopy( - getProfileFunctionSummaries( - snapshot.profiles as unknown as Parameters[0], - filter, - ), - ), - getQuerySummaries: filter => - readonlyCopy( - getQuerySummaries(getSpans(filteredTraces(filter) as Trace[]), { description: filter?.description }), - ), - getResourceSummaries: filter => readonlyCopy(getResourceSummaries(getSpans(filteredTraces(filter) as Trace[]))), - getAITraces: filter => readonlyCopy(getAITraces(filteredTraces(filter) as Trace[])), - getWebVitals: () => readonlyCopy(getWebVitals(snapshot.events as SentryEvent[])), - getSdks: () => snapshot.sdks, - subscribe: listener => { + drain: () => queue.then(() => {}), + getStats: () => + Object.freeze({ + pending, + retainedEnvelopes: journal.length, + retainedBytes: journal.reduce((sum, item) => sum + item.bytes, 0), + revision, + disposed, + }), + subscribe(listener) { if (disposed) return () => {}; listeners.add(listener); return () => { listeners.delete(listener); }; }, - reset: () => { + reset() { if (disposed) return; generation++; + lifetime.abort(); + lifetime = new AbortController(); clearTimer(); engine.store.getState().resetData(); journal = []; enrichmentCache = new Map(); engine = makeEngine(); queue = Promise.resolve(); - publish(); + pruning = undefined; + pending = 0; + publish("reset"); }, - prune: () => - enqueue(async epoch => { + prune() { + if (disposed) return Promise.resolve(); + if (pruning) return pruning; + // Control work has one coalesced continuation outside ingestion admission. + // Link it into the same barrier so even invalid/empty work cannot skip expiry. + const epoch = generation; + const operation = queue.then(async () => { if (!active(epoch)) return; - if (await enforceRetention(epoch)) { - if (active(epoch)) publish(); - } - scheduleExpiry(); - }), - dispose: () => { + currentDiagnostics = []; + const evicted = await enforceRetention(epoch); + if (active(epoch) && evicted.length) publish("evict", evicted); + if (active(epoch)) scheduleExpiry(); + }); + queue = operation.catch(() => {}); + pruning = operation; + void operation.then( + () => { + if (pruning === operation) pruning = undefined; + }, + () => { + if (pruning === operation) pruning = undefined; + }, + ); + return operation; + }, + dispose() { if (disposed) return; generation++; disposed = true; + lifetime.abort(); clearTimer(); engine.store.getState().resetData(); journal = []; enrichmentCache.clear(); - listeners.clear(); engine = makeEngine(); - revision++; - snapshot = stateSnapshot(); + pending = 0; + publish("dispose"); + listeners.clear(); + pruning = undefined; queue = Promise.resolve(); }, }; return client; } - -export type { Ingester, IngesterOptions, IngesterSnapshot, IngestResult, Diagnostic } from "./public.ts"; +export type { Ingester, IngesterOptions, IngesterSnapshot } from "./public.ts"; +export type { IngestResult, Diagnostic } from "./contracts.ts"; diff --git a/packages/spotlight/src/telemetry/internal/envelopes.ts b/packages/spotlight/src/telemetry/internal/envelopes.ts index 2042e2f58..808479f89 100644 --- a/packages/spotlight/src/telemetry/internal/envelopes.ts +++ b/packages/spotlight/src/telemetry/internal/envelopes.ts @@ -1,5 +1,6 @@ import type { Envelope } from "@sentry/core"; import { SUPPORTED_EVENT_TYPES } from "../constants/sentry.ts"; +import { opaquePayload } from "../envelope/payload.ts"; export function isRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); @@ -19,36 +20,14 @@ export function validateEnvelope(input: unknown): input is Envelope { ); } -const TEXT_CONTENT_TYPES = new Set([ - "text/plain", - "text/css", - "text/csv", - "text/html", - "text/javascript", - "text/json", - "text/x-json", - "application/json", - "application/ld+json", -]); - -/** Match the wire decoder's attachment representation without requiring Node's Buffer. */ +/** Match serialized opaque items with the shared portable byte policy. */ export function normalizeEnvelope(input: Envelope): Envelope { const envelope = structuredClone(input); for (const item of envelope[1]) { const [header, payload] = item; if (typeof payload !== "string" && !(payload instanceof Uint8Array)) continue; - const bytes = typeof payload === "string" ? new TextEncoder().encode(payload) : payload; - let data: string; - if (TEXT_CONTENT_TYPES.has(header.content_type as string)) { - data = new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); - } else { - let binary = ""; - for (let offset = 0; offset < bytes.length; offset += 0x8000) { - binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); - } - data = btoa(binary); - } - item[1] = { data } as (typeof item)[1]; + if (!header.content_type && (header.type as string) === "statsd") header.content_type = "text/plain"; + item[1] = opaquePayload(payload, header.content_type as string) as (typeof item)[1]; } return envelope; } diff --git a/packages/spotlight/src/telemetry/internal/results.ts b/packages/spotlight/src/telemetry/internal/results.ts new file mode 100644 index 000000000..6dbc8c20f --- /dev/null +++ b/packages/spotlight/src/telemetry/internal/results.ts @@ -0,0 +1,35 @@ +import type { Diagnostic, DiagnosticCode, IngestCounts, IngestOutcome, IngestResult } from "../contracts.ts"; + +export const EMPTY_COUNTS: IngestCounts = Object.freeze({ + events: 0, + spans: 0, + logs: 0, + skippedItems: 0, + duplicateEvents: 0, +}); +export function diagnostic( + code: DiagnosticCode, + message: string, + details: Partial> = {}, +): Diagnostic { + return Object.freeze({ code, message, severity: "error", ...details }); +} +export function reportDiagnostic(observer: ((item: Diagnostic) => void) | undefined, item: Diagnostic) { + try { + observer?.(item); + } catch { + /* Observers never change capture outcomes. */ + } +} +export function receipt( + outcome: IngestOutcome, + metadata: { revision: number; retained?: boolean; counts?: Partial }, +): IngestResult { + return Object.freeze({ + ...outcome, + diagnostics: Object.freeze((outcome.diagnostics ?? []).map(item => Object.freeze({ ...item }))), + revision: metadata.revision, + retained: metadata.retained ?? false, + counts: Object.freeze({ ...EMPTY_COUNTS, ...metadata.counts }), + }) as IngestResult; +} diff --git a/packages/spotlight/src/telemetry/lifecycle.spec.ts b/packages/spotlight/src/telemetry/lifecycle.spec.ts new file mode 100644 index 000000000..06915c4d9 --- /dev/null +++ b/packages/spotlight/src/telemetry/lifecycle.spec.ts @@ -0,0 +1,231 @@ +// @vitest-environment node +import { afterEach, expect, test, vi } from "vitest"; +import { createPipeline } from "../sdk/node.ts"; +import { createIngester } from "./index.ts"; + +const event = (id = "event") => [{}, [[{ type: "event" }, { event_id: id, timestamp: 1 }]]]; +const error = (id = "error") => [ + {}, + [[{ type: "event" }, { event_id: id, timestamp: 1, exception: { values: [{ type: "Error", value: "original" }] } }]], +]; +afterEach(() => vi.useRealTimers()); + +test.each(["core", "node"] as const)("%s pruning after full-queue no-op work still enforces expiry", async runtime => { + vi.useFakeTimers(); + const client = + runtime === "core" + ? createIngester({ maxPending: 1, retention: { maxAgeMs: 1000 } }) + : createPipeline({ maxPending: 1, retention: { maxAgeMs: 1000 } }); + await client.ingestEnvelope(event()); + vi.setSystemTime(Date.now() + 10000); + const work = "ingestPayload" in client ? client.ingestPayload({ body: "bad" }) : client.ingestBatch([]); + await Promise.all([work, client.prune()]); + expect(client.getEnvelopes()).toEqual([]); + expect(client.getStats().pending).toBe(0); + client.dispose(); +}); + +test("pruning is coalesced and cannot bypass the configured pending limit", async () => { + const ingester = createIngester({ maxPending: 1 }); + const operations = Array.from({ length: 100 }, () => ingester.prune()); + expect(ingester.getStats().pending).toBeLessThanOrEqual(1); + expect(operations[0]).toBe(operations[99]); + await Promise.all(operations); + expect(ingester.getStats().pending).toBe(0); + ingester.dispose(); +}); + +test("large enrichment deadlines do not overflow Node timers", async () => { + vi.useFakeTimers(); + const ingester = createIngester({ + enrichmentTimeoutMs: 2147483648, + enrichStacktrace: async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + }, + }); + const pending = ingester.ingestEnvelope(error()); + await vi.advanceTimersByTimeAsync(10); + expect(await pending).toMatchObject({ status: "processed", diagnostics: [] }); + ingester.dispose(); +}); + +test("receipts distinguish retained unsupported data and no-op duplicates", async () => { + const ingester = createIngester(); + const first = await ingester.ingestEnvelope(event(), { deliveryId: "delivery" }); + expect(first).toMatchObject({ + status: "processed", + envelopeId: "delivery", + retained: true, + revision: 1, + counts: { events: 1, duplicateEvents: 0 }, + }); + expect(await ingester.ingestEnvelope(event(), { deliveryId: "delivery" })).toMatchObject({ + status: "ignored", + reason: "duplicate", + retained: true, + revision: 1, + }); + expect(await ingester.ingestEnvelope([{}, [[{ type: "future" }, {}]]])).toMatchObject({ + status: "ignored", + reason: "unsupported_items", + retained: true, + revision: 2, + counts: { skippedItems: 1 }, + }); + expect(Object.isFrozen(first.counts)).toBe(true); + ingester.dispose(); +}); + +test("batch ingestion preserves order and a drain barrier covers earlier work", async () => { + const ingester = createIngester(); + const batch = ingester.ingestBatch([event("one"), null, event("two")]); + await ingester.drain(); + expect((await batch).map(result => result.status)).toEqual(["processed", "invalid", "processed"]); + expect(ingester.getEvents().map(item => item.event_id)).toEqual(["one", "two"]); + expect(ingester.getStats().pending).toBe(0); + ingester.dispose(); +}); + +test("batches are one queued operation and drain does not wait for later submissions", async () => { + let releaseFirst!: () => void; + let releaseLater!: () => void; + let enteredFirst!: () => void; + const ready = new Promise(resolve => { + enteredFirst = resolve; + }); + const ingester = createIngester({ + enrichStacktrace: async item => { + if (item.event_id === "first") { + enteredFirst(); + await new Promise(resolve => { + releaseFirst = resolve; + }); + } + if (item.event_id === "later") + await new Promise(resolve => { + releaseLater = resolve; + }); + }, + }); + const batch = ingester.ingestBatch([error("first"), event("second")]); + const barrier = ingester.drain(); + const later = ingester.ingestEnvelope(error("later")); + await ready; + releaseFirst(); + await barrier; + expect((await batch).map(result => result.status)).toEqual(["processed", "processed"]); + expect(ingester.getEvent("second")).toBeDefined(); + // The later hook may start, but it cannot keep the earlier barrier pending. + await vi.waitFor(() => expect(releaseLater).toBeTypeOf("function")); + releaseLater(); + await later; + ingester.dispose(); +}); + +test("a timed out hook cannot mutate committed telemetry or block later work", async () => { + vi.useFakeTimers(); + let finish!: () => void; + const ingester = createIngester({ + enrichmentTimeoutMs: 20, + enrichStacktrace: async item => { + await new Promise(resolve => { + finish = resolve; + }); + item.exception.values![0].value = "late"; + }, + }); + const captured = ingester.ingestEnvelope(error()); + const later = ingester.ingestEnvelope(event("later")); + await vi.advanceTimersByTimeAsync(20); + expect(await captured).toMatchObject({ + status: "processed", + diagnostics: [{ code: "enrichment_timeout", severity: "warning" }], + }); + expect((await later).status).toBe("processed"); + finish(); + await Promise.resolve(); + expect(ingester.getErrors()[0].exception.values![0].value).toBe("original"); + ingester.dispose(); +}); + +test("queue capacity and abort have structured outcomes and release capacity", async () => { + let entered!: () => void; + const ready = new Promise(resolve => { + entered = resolve; + }); + const ingester = createIngester({ + maxPending: 1, + enrichStacktrace: async (_item, { signal }) => { + entered(); + await new Promise(resolve => signal.addEventListener("abort", () => resolve(), { once: true })); + }, + }); + const controller = new AbortController(); + const active = ingester.ingestEnvelope(error(), { signal: controller.signal }); + await ready; + expect(await ingester.ingestEnvelope(event())).toMatchObject({ + status: "ignored", + reason: "queue_full", + retained: false, + }); + controller.abort(); + expect(await active).toMatchObject({ status: "ignored", reason: "aborted" }); + expect(ingester.getEvents()).toEqual([]); + expect((await ingester.ingestEnvelope(event())).status).toBe("processed"); + ingester.dispose(); +}); + +test("abort during post-ingestion retention replay restores the prior committed history", async () => { + let replaying!: () => void; + const replayReady = new Promise(resolve => { + replaying = resolve; + }); + let calls = 0; + let replaySignal: AbortSignal | undefined; + const ingester = createIngester({ + retention: { maxEnvelopes: 1 }, + enrichStacktrace: async (_event, { signal }) => { + if (++calls === 1) return; + replaySignal = signal; + replaying(); + await new Promise(resolve => signal.addEventListener("abort", () => resolve(), { once: true })); + }, + }); + await ingester.ingestEnvelope(error("same"), { deliveryId: "one" }); + const snapshot = ingester.getSnapshot(); + const controller = new AbortController(); + const aborted = ingester.ingestEnvelope(error("same"), { deliveryId: "two", signal: controller.signal }); + await replayReady; + controller.abort(); + expect(await aborted).toMatchObject({ status: "ignored", reason: "aborted", retained: false }); + expect(replaySignal?.aborted).toBe(true); + expect(ingester.getSnapshot()).toBe(snapshot); + expect(ingester.getEnvelope("one")).toBeDefined(); + expect(ingester.getEnvelope("two")).toBeUndefined(); + expect(calls).toBe(2); + ingester.dispose(); +}); + +test("byte retention evicts dependent data and delivery keys have the same lifetime", async () => { + const ingester = createIngester({ retention: { maxBytes: 100 } }); + await ingester.ingestEnvelope(event("one"), { deliveryId: "one" }); + await ingester.ingestEnvelope(event("two"), { deliveryId: "two" }); + expect(ingester.getStats().retainedBytes).toBeLessThanOrEqual(100); + expect(ingester.getEvent("one")).toBeUndefined(); + expect((await ingester.ingestEnvelope(event("one"), { deliveryId: "one" })).status).toBe("processed"); + ingester.dispose(); +}); + +test("changes identify ingestion, eviction and reset while callbacks see committed reads", async () => { + const ingester = createIngester({ retention: { maxEnvelopes: 1 } }); + const changes: string[] = []; + ingester.subscribe((snapshot, change) => { + expect(snapshot).toBe(ingester.getSnapshot()); + changes.push(change.type); + }); + await ingester.ingestEnvelope(event("one")); + await ingester.ingestEnvelope(event("two")); + ingester.reset(); + expect(changes).toEqual(["ingest", "ingest", "reset"]); + ingester.dispose(); +}); diff --git a/packages/spotlight/src/telemetry/parity.spec.ts b/packages/spotlight/src/telemetry/parity.spec.ts index 0e2fbf869..75ba59149 100644 --- a/packages/spotlight/src/telemetry/parity.spec.ts +++ b/packages/spotlight/src/telemetry/parity.spec.ts @@ -10,7 +10,7 @@ import { getAITraces, getQuerySummaries, getSpans } from "./selectors/index.ts"; test("the UI store, browser processor, and Node SDK derive identical traces and queries", async () => { const spotlight = createPipeline(); const browser = createIngester(); - const result = await spotlight.ingest({ + const result = await spotlight.ingestPayload({ body: readFileSync(new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url)), contentType: "application/x-sentry-envelope", }); @@ -23,7 +23,7 @@ test("the UI store, browser processor, and Node SDK derive identical traces and const uiTraces = useSentryStore.getState().getTraces(); expect(uiTraces).toEqual(spotlight.getTraces()); expect(browser.getTraces()).toEqual(spotlight.getTraces()); - expect(getQuerySummaries(getSpans(uiTraces))).toEqual(spotlight.getQuerySummaries()); + expect(getQuerySummaries(getSpans(uiTraces))).toEqual(spotlight.getQuerySummaries({ includeSpans: true })); expect(browser.getQuerySummaries()).toEqual(spotlight.getQuerySummaries()); expect(JSON.stringify(envelope)).toBe(original); useSentryStore.getState().resetData(); @@ -34,7 +34,7 @@ test("the UI, browser processor, and Node SDK share log, profile, and AI process const browser = createIngester(); useSentryStore.getState().resetData(); for (const file of ["continuous_profiling/transaction.txt", "continuous_profiling/profile_chunk_1.txt"]) { - const result = await spotlight.ingest({ + const result = await spotlight.ingestPayload({ body: readFileSync(new URL(`../../_fixtures/${file}`, import.meta.url)), contentType: "application/x-sentry-envelope", }); diff --git a/packages/spotlight/src/telemetry/public.ts b/packages/spotlight/src/telemetry/public.ts index bf80912e8..29c59639f 100644 --- a/packages/spotlight/src/telemetry/public.ts +++ b/packages/spotlight/src/telemetry/public.ts @@ -1,4 +1,13 @@ -import type { Envelope } from "@sentry/core"; +import type { Envelope, EnvelopeItem } from "@sentry/core"; +import type { + BatchOptions, + ChangeMetadata, + Diagnostic, + IngestOptions, + IngestResult, + IngestStats, +} from "./contracts.ts"; +import type { TelemetryReader } from "./reader.ts"; import type { ProfileFunctionSummary as InternalProfileFunctionSummary, QuerySummary as InternalQuerySummary, @@ -10,7 +19,6 @@ import type { Sdk, SentryErrorEvent, SentryEvent, - SentryEventWithPerformanceData, SentryLogEventItem, SentryProfileWithTraceMeta, Span, @@ -30,7 +38,9 @@ export type DeepReadonly = T extends (...args: never[]) => unknown : T; /** A parsed Sentry wire envelope; input timestamps retain their wire units. */ -export type RawEnvelope = Envelope; +// Item payloads remain unknown: forward-compatible wire records include span-v2 +// containers that older @sentry/core payload unions cannot represent. +export type RawEnvelope = [Envelope[0], [EnvelopeItem[0], unknown][]]; export type ProcessedEvent = DeepReadonly; export type ProcessedSpan = DeepReadonly; export type ProcessedTrace = DeepReadonly; @@ -40,28 +50,29 @@ export type AITrace = DeepReadonly; export type QuerySummary = DeepReadonly; export type ResourceSummary = DeepReadonly; export type ProfileFunctionSummary = DeepReadonly; -export type Diagnostic = { readonly code: string; readonly message: string }; -export type IngestResult = - | { readonly status: "processed"; readonly envelopeId: string; readonly diagnostics: readonly Diagnostic[] } - | { - readonly status: "ignored"; - readonly reason: string; - readonly envelopeId?: string; - readonly diagnostics: readonly Diagnostic[]; - } - | { readonly status: "invalid"; readonly diagnostics: readonly Diagnostic[] }; -export type TraceFilter = { traceId?: string }; -export type EventFilter = TraceFilter & { spanId?: string }; -export type DatabaseQueryFilter = TraceFilter & { description?: string }; +export type * from "./contracts.ts"; +export type { + TraceReadFilter as TraceFilter, + EventReadFilter as EventFilter, + DatabaseReadFilter as DatabaseQueryFilter, +} from "./reader.ts"; export type RetentionOptions = { /** Maximum retained envelopes, including all telemetry derived from them. Default: 500. */ maxEnvelopes?: number; /** Optional maximum age since receipt. Expiration runs automatically; dispose releases the timer. */ maxAgeMs?: number; + /** Serialized retained-envelope budget. Default: 64 MiB. */ + maxBytes?: number; }; export type IngesterOptions = { retention?: RetentionOptions; - enrichStacktrace?: (event: SentryErrorEvent) => Promise; + enrichStacktrace?: (event: SentryErrorEvent, context: { signal: AbortSignal }) => Promise; + /** Maximum ingestion operations, including active work. Default: 64. Overflow returns ignored/queue_full. */ + maxPending?: number; + /** Maximum serialized parsed envelope size. Default: 32 MiB. */ + maxEnvelopeBytes?: number; + /** Optional enrichment deadline. Default: 5000 ms. */ + enrichmentTimeoutMs?: number; onDiagnostic?: (diagnostic: Diagnostic) => void; }; export type IngesterSnapshot = { @@ -73,27 +84,13 @@ export type IngesterSnapshot = { readonly sdks: readonly DeepReadonly[]; readonly profiles: ReadonlyMap; }; -export type Ingester = { - ingestEnvelope: (input: unknown) => Promise; - getSnapshot: () => IngesterSnapshot; - getEnvelope: (id: string) => DeepReadonly | undefined; - getEnvelopes: () => readonly DeepReadonly[]; - getEvent: (id: string) => ProcessedEvent | undefined; - getEvents: (filter?: EventFilter) => readonly ProcessedEvent[]; - getErrors: (filter?: EventFilter) => readonly DeepReadonly[]; - getTrace: (id: string) => ProcessedTrace | undefined; - getTraces: (filter?: TraceFilter) => readonly ProcessedTrace[]; - getSpans: (filter?: TraceFilter) => readonly ProcessedSpan[]; - getLog: (id: string) => ProcessedLog | undefined; - getLogs: (filter?: TraceFilter) => readonly ProcessedLog[]; - getProfile: (traceId: string) => ProcessedProfile | undefined; - getProfileFunctionSummaries: (filter?: TraceFilter) => readonly ProfileFunctionSummary[]; - getQuerySummaries: (filter?: DatabaseQueryFilter) => readonly QuerySummary[]; - getResourceSummaries: (filter?: TraceFilter) => readonly ResourceSummary[]; - getAITraces: (filter?: TraceFilter) => readonly AITrace[]; - getWebVitals: () => readonly DeepReadonly[]; - getSdks: () => readonly DeepReadonly[]; - subscribe: (listener: (snapshot: IngesterSnapshot) => void) => () => void; +export type Ingester = TelemetryReader & { + ingestEnvelope: (input: unknown, options?: IngestOptions) => Promise; + ingestBatch: (inputs: Iterable, options?: BatchOptions) => Promise; + /** Wait for operations submitted before this call; future operations are excluded. */ + drain: () => Promise; + getStats: () => IngestStats; + subscribe: (listener: (snapshot: IngesterSnapshot, change: ChangeMetadata) => void) => () => void; reset: () => void; prune: () => Promise; dispose: () => void; diff --git a/packages/spotlight/src/telemetry/reader.spec.ts b/packages/spotlight/src/telemetry/reader.spec.ts new file mode 100644 index 000000000..31f5bbb86 --- /dev/null +++ b/packages/spotlight/src/telemetry/reader.spec.ts @@ -0,0 +1,180 @@ +// @vitest-environment node +import { expect, test } from "vitest"; +import type { IngesterSnapshot } from "./public.ts"; +import { createTelemetryReader } from "./reader.ts"; +import type { Span, Trace } from "./types.ts"; +function fixture() { + const spans: Span[] = [ + { span_id: "b", start_timestamp: 10, timestamp: 20, op: "db.query", description: "SELECT 1" }, + { span_id: "a", start_timestamp: 5, timestamp: 20, op: "db.query", description: "SELECT 1" }, + { + span_id: "c", + start_timestamp: 25, + timestamp: 30, + op: "resource.script", + description: "script.js", + data: { "http.response_content_length": 40 }, + }, + ]; + const trace: Trace = { + trace_id: "trace", + transactions: [], + errors: 0, + start_timestamp: 5, + timestamp: 30, + rootTransaction: null, + rootTransactionName: "", + spans: new Map(spans.map(span => [span.span_id, span])), + spanTree: spans, + profileGrafted: false, + }; + const snapshot: IngesterSnapshot = { + revision: 1, + envelopes: [], + events: [ + { event_id: "b", timestamp: 20, exception: { value: undefined, values: [] } }, + { event_id: "a", timestamp: 20, exception: { value: undefined, values: [] } }, + { event_id: "c", timestamp: 30, exception: { value: undefined, values: [] } }, + ], + traces: [trace], + logs: [], + sdks: [], + profiles: new Map(), + }; + return { snapshot, spans }; +} +test("inclusive milliseconds and deterministic IDs apply without truncating defaults", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + expect(reader.getEvents({ fromMs: 20, toMs: 20 }).map(event => event.event_id)).toEqual(["a", "b"]); + expect(reader.getEvents({ order: "desc", limit: 2 }).map(event => event.event_id)).toEqual(["c", "b"]); + expect(reader.getSpans({ traceId: "trace", fromMs: 20, toMs: 20 }).map(span => span.span_id)).toEqual(["a", "b"]); + expect(reader.getSpans({ traceId: "other" })).toEqual([]); + const many = { + ...snapshot, + events: Array.from({ length: 1200 }, (_, n) => ({ + event_id: String(n), + timestamp: n, + exception: { value: undefined, values: [] }, + })), + }; + expect(createTelemetryReader(() => many).getEvents()).toHaveLength(1200); +}); +test("invalid getter filters throw", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + for (const filter of [ + { fromMs: Number.NaN }, + { fromMs: 20, toMs: 10 }, + { limit: 0 }, + { limit: 1.5 }, + { order: "sideways" }, + ]) + expect(() => reader.getEvents(filter as Parameters[0])).toThrow(RangeError); +}); +test("summaries default to compact and preserve trace-less legacy spans", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + expect(reader.getQuerySummaries({ traceId: "trace", toMs: 20 })).toEqual([ + { description: "SELECT 1", count: 2, totalDurationMs: 25, averageDurationMs: 12.5 }, + ]); + expect(reader.getQuerySummaries({ includeSpans: true })[0].spans).toHaveLength(2); + expect(reader.getResourceSummaries()).toEqual([ + { description: "script.js", count: 1, totalDurationMs: 5, averageDurationMs: 5, averageEncodedSizeBytes: 40 }, + ]); + expect(reader.getResourceSummaries({ includeSpans: true })[0].spans).toHaveLength(1); +}); +test("canonical filters share cached identity until revision changes", () => { + let { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + const first = reader.getQuerySummaries({ traceId: "trace", fromMs: 10 }); + expect(reader.getQuerySummaries({ fromMs: 10, traceId: "trace" })).toBe(first); + expect(reader.getEvents()).toBe(reader.getEvents({})); + snapshot = { ...snapshot, revision: 2 }; + expect(reader.getQuerySummaries({ fromMs: 10, traceId: "trace" })).not.toBe(first); +}); +test("page cursors bind collection, canonical query and revision", () => { + let { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + const first = reader.readPage("events", { limit: 1, fromMs: 20 }); + expect(first).toMatchObject({ status: "ok", revision: 1, total: 3, items: [{ event_id: "a" }] }); + if (first.status !== "ok") throw new Error("expected page"); + expect(reader.readPage("events", { fromMs: 20, limit: 1, cursor: first.nextCursor })).toMatchObject({ + status: "ok", + items: [{ event_id: "b" }], + }); + expect(reader.readPage("logs", { fromMs: 20, limit: 1, cursor: first.nextCursor })).toMatchObject({ + status: "invalid", + code: "invalid_cursor", + }); + expect(reader.readPage("events", { fromMs: 21, limit: 1, cursor: first.nextCursor })).toMatchObject({ + status: "invalid", + code: "invalid_cursor", + }); + snapshot = { ...snapshot, revision: 2 }; + expect(reader.readPage("events", { fromMs: 20, limit: 1, cursor: first.nextCursor })).toMatchObject({ + status: "invalid", + code: "stale_cursor", + }); + expect(reader.readPage("events", { cursor: "bad" })).toMatchObject({ status: "invalid", code: "invalid_cursor" }); + expect(reader.readPage("events", { limit: 1001 })).toMatchObject({ status: "invalid", code: "invalid_query" }); + expect(reader.readPage("events", { fromMs: 30, toMs: 20 })).toMatchObject({ + status: "invalid", + code: "invalid_query", + }); +}); +test("pages default to 100 and memoize result", () => { + const { snapshot } = fixture(); + const many = { + ...snapshot, + events: Array.from({ length: 1200 }, (_, n) => ({ + event_id: String(n), + timestamp: n, + exception: { value: undefined, values: [] }, + })), + }; + const reader = createTelemetryReader(() => many); + const page = reader.readPage("events"); + expect(page.status === "ok" && page.items.length).toBe(100); + expect(reader.readPage("events", {})).toBe(page); +}); + +test("explicit default filter values share cached results", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + expect(reader.getEvents({ order: "asc" })).toBe(reader.getEvents()); + expect(reader.getQuerySummaries({ includeSpans: false })).toBe(reader.getQuerySummaries()); +}); + +test("web vitals apply limit after selecting measurements", () => { + const { snapshot } = fixture(); + const measured = { + event_id: "vital", + timestamp: 40, + type: "transaction" as const, + start_timestamp: 30, + contexts: { trace: { trace_id: "trace", op: "pageload" } }, + measurements: { + fcp: { value: 100, unit: "millisecond" }, + lcp: { value: 100, unit: "millisecond" }, + cls: { value: 0, unit: "" }, + ttfb: { value: 50, unit: "millisecond" }, + fid: { value: 10, unit: "millisecond" }, + }, + }; + const reader = createTelemetryReader(() => ({ ...snapshot, events: [...snapshot.events, measured] })); + expect(reader.getWebVitals({ limit: 1 }).map(event => event.event_id)).toEqual(["vital"]); +}); + +test("reader bounds cached queries while retaining recent identities", () => { + const { snapshot } = fixture(); + const reader = createTelemetryReader(() => snapshot); + const oldest = reader.getEvents({ fromMs: -1 }); + const recent = reader.getEvents({ fromMs: 0 }); + for (let index = 1; index <= 260; index++) { + reader.getEvents({ fromMs: index }); + expect(reader.getEvents({ fromMs: 0 })).toBe(recent); + } + expect(reader.getEvents({ fromMs: -1 })).not.toBe(oldest); + expect(reader.readPage("events", { limit: 0 })).toMatchObject({ revision: 1 }); +}); diff --git a/packages/spotlight/src/telemetry/reader.ts b/packages/spotlight/src/telemetry/reader.ts new file mode 100644 index 000000000..78089c4a6 --- /dev/null +++ b/packages/spotlight/src/telemetry/reader.ts @@ -0,0 +1,407 @@ +import { readonlyCopy } from "./internal/readonly.ts"; +import type { + AITrace, + DeepReadonly, + IngesterSnapshot, + ProcessedEvent, + ProcessedLog, + ProcessedProfile, + ProcessedSpan, + ProcessedTrace, + ProfileFunctionSummary, + QuerySummary, + ResourceSummary, +} from "./public.ts"; +import { + getAITraces, + getProfileFunctionSummaries, + getQuerySummaries, + getResourceSummaries, + getWebVitals, +} from "./selectors/index.ts"; +import type { Sdk, SentryErrorEvent, SentryEvent, SentryEventWithPerformanceData, Span, Trace } from "./types.ts"; +import { isErrorEvent } from "./utils/sentry.ts"; + +/** Inclusive bounds in Unix milliseconds. No implicit limit is applied to getters. */ +export type TelemetryReadFilter = { + readonly fromMs?: number; + readonly toMs?: number; + readonly order?: "asc" | "desc"; + readonly limit?: number; +}; +export type TraceReadFilter = TelemetryReadFilter & { readonly traceId?: string }; +export type EventReadFilter = TraceReadFilter & { readonly spanId?: string }; +export type SummaryReadFilter = TraceReadFilter & { readonly includeSpans?: boolean }; +export type DatabaseReadFilter = SummaryReadFilter & { readonly description?: string }; +export type CompactQuerySummary = Omit; +export type CompactResourceSummary = Omit; +export type ReadCollection = "events" | "traces" | "logs" | "spans"; +export type ReadPageOptions = EventReadFilter & { readonly cursor?: string }; +export type ReadCollectionItem = { + events: ProcessedEvent; + traces: ProcessedTrace; + logs: ProcessedLog; + spans: ProcessedSpan; +}[C]; +export type ReadPageResult = + | { + readonly status: "ok"; + readonly revision: number; + readonly items: readonly T[]; + readonly total: number; + readonly nextCursor?: string; + } + | { + readonly status: "invalid"; + readonly revision: number; + readonly code: "stale_cursor" | "invalid_cursor" | "invalid_query"; + readonly message: string; + }; +export type TelemetryReader = { + getSnapshot(): IngesterSnapshot; + getEnvelope(id: string): IngesterSnapshot["envelopes"][number] | undefined; + getEnvelopes(): IngesterSnapshot["envelopes"]; + getEvent(id: string): ProcessedEvent | undefined; + getEvents(filter?: EventReadFilter): readonly ProcessedEvent[]; + getErrors(filter?: EventReadFilter): readonly DeepReadonly[]; + getTrace(id: string): ProcessedTrace | undefined; + getTraces(filter?: TraceReadFilter): readonly ProcessedTrace[]; + getSpans(filter?: TraceReadFilter): readonly ProcessedSpan[]; + getLog(id: string): ProcessedLog | undefined; + getLogs(filter?: TraceReadFilter): readonly ProcessedLog[]; + getProfile(traceId: string): ProcessedProfile | undefined; + getProfileFunctionSummaries(filter?: TraceReadFilter): readonly ProfileFunctionSummary[]; + getQuerySummaries(filter: DatabaseReadFilter & { includeSpans: true }): readonly QuerySummary[]; + getQuerySummaries(filter?: DatabaseReadFilter & { includeSpans?: false }): readonly CompactQuerySummary[]; + getQuerySummaries( + filter?: DatabaseReadFilter, + ): readonly (CompactQuerySummary & { readonly spans?: QuerySummary["spans"] })[]; + getResourceSummaries(filter: SummaryReadFilter & { includeSpans: true }): readonly ResourceSummary[]; + getResourceSummaries(filter?: SummaryReadFilter & { includeSpans?: false }): readonly CompactResourceSummary[]; + getResourceSummaries( + filter?: SummaryReadFilter, + ): readonly (CompactResourceSummary & { readonly spans?: ResourceSummary["spans"] })[]; + getAITraces(filter?: TraceReadFilter): readonly AITrace[]; + getWebVitals(filter?: EventReadFilter): readonly DeepReadonly[]; + getSdks(): readonly DeepReadonly[]; + readPage(collection: C, options?: ReadPageOptions): ReadPageResult>; +}; + +function validate(filter: ReadPageOptions, paged = false) { + for (const value of [filter.fromMs, filter.toMs]) + if (value !== undefined && !Number.isFinite(value)) + throw new RangeError("Time bounds must be finite Unix milliseconds"); + if (filter.fromMs !== undefined && filter.toMs !== undefined && filter.fromMs > filter.toMs) + throw new RangeError("fromMs must not exceed toMs"); + if (filter.order !== undefined && filter.order !== "asc" && filter.order !== "desc") + throw new RangeError("order must be asc or desc"); + if ( + filter.limit !== undefined && + (!Number.isSafeInteger(filter.limit) || filter.limit < 1 || (paged && filter.limit > 1000)) + ) + throw new RangeError(paged ? "Page limit must be between 1 and 1000" : "limit must be a positive safe integer"); + for (const value of [filter.traceId, filter.spanId]) + if (value !== undefined && typeof value !== "string") throw new RangeError("IDs must be strings"); +} +function canonical(value: object): string { + return JSON.stringify( + Object.fromEntries( + Object.entries(value) + .filter( + ([key, item]) => + item !== undefined && !(key === "order" && item === "asc") && !(key === "includeSpans" && item === false), + ) + .sort(([a], [b]) => a.localeCompare(b)), + ), + ); +} +function within(timestamp: number, filter: TelemetryReadFilter) { + return ( + (filter.fromMs === undefined || timestamp >= filter.fromMs) && + (filter.toMs === undefined || timestamp <= filter.toMs) + ); +} +function ordered( + items: readonly T[], + filter: TelemetryReadFilter, + time: (item: T) => number, + id: (item: T) => string, +): readonly T[] { + const direction = filter.order === "desc" ? -1 : 1; + return Object.freeze( + items + .filter(item => within(time(item), filter)) + .sort((a, b) => direction * (time(a) - time(b) || (id(a) < id(b) ? -1 : id(a) > id(b) ? 1 : 0))) + .slice(0, filter.limit), + ); +} +function named(items: readonly T[], filter: TelemetryReadFilter, name: (item: T) => string): readonly T[] { + return ordered(items, { order: filter.order, limit: filter.limit }, () => 0, name); +} + +/** A portable read-only dependency. The current revision retains at most 256 recently used reads. */ +export function createTelemetryReader(getSnapshot: () => IngesterSnapshot): TelemetryReader { + let revision = -1; + const cache = new Map(); + function read(name: string, filter: object, derive: (snapshot: IngesterSnapshot) => T): T { + const snapshot = getSnapshot(); + if (revision !== snapshot.revision) { + cache.clear(); + revision = snapshot.revision; + } + const key = `${name}:${canonical(filter)}`; + if (cache.has(key)) { + const value = cache.get(key) as T; + cache.delete(key); + cache.set(key, value); + return value; + } + const value = derive(snapshot); + cache.set(key, value); + if (cache.size > 256) cache.delete(cache.keys().next().value!); + return value; + } + const traces = (snapshot: IngesterSnapshot, filter: TraceReadFilter) => + snapshot.traces.filter(trace => filter.traceId === undefined || trace.trace_id === filter.traceId); + // Legacy transaction child spans need not carry trace_id. Scope their owning trace first. + const spans = (snapshot: IngesterSnapshot, filter: TraceReadFilter) => + traces(snapshot, filter).flatMap(trace => Array.from(trace.spans.values())); + function events(snapshot: IngesterSnapshot, filter: EventReadFilter) { + return snapshot.events.filter( + event => + (filter.traceId === undefined || + event.contexts?.trace?.trace_id === filter.traceId || + (event.type === "span" && event.span.trace_id === filter.traceId)) && + (filter.spanId === undefined || + event.contexts?.trace?.span_id === filter.spanId || + (event.type === "span" && event.span.span_id === filter.spanId)), + ); + } + const reader: TelemetryReader = { + getSnapshot, + getEnvelope: id => getSnapshot().envelopes.find(envelope => String(envelope[0].__spotlight_envelope_id) === id), + getEnvelopes: () => getSnapshot().envelopes, + getEvent: id => getSnapshot().events.find(event => event.event_id === id), + getEvents(filter = {}) { + validate(filter); + return read("events", filter, snapshot => + ordered( + events(snapshot, filter), + filter, + event => event.timestamp, + event => event.event_id, + ), + ); + }, + getErrors(filter = {}) { + validate(filter); + return read("errors", filter, snapshot => + ordered( + events(snapshot, filter).filter(event => + isErrorEvent(event as SentryEvent), + ) as readonly DeepReadonly[], + filter, + event => event.timestamp, + event => event.event_id, + ), + ); + }, + getTrace: id => getSnapshot().traces.find(trace => trace.trace_id === id), + getTraces(filter = {}) { + validate(filter); + return read("traces", filter, snapshot => + ordered( + traces(snapshot, filter), + filter, + trace => trace.timestamp, + trace => trace.trace_id, + ), + ); + }, + getSpans(filter = {}) { + validate(filter); + return read("spans", filter, snapshot => + ordered( + spans(snapshot, filter), + filter, + span => span.timestamp, + span => span.span_id, + ), + ); + }, + getLog: id => getSnapshot().logs.find(log => log.id === id), + getLogs(filter = {}) { + validate(filter); + return read("logs", filter, snapshot => + ordered( + snapshot.logs.filter(log => filter.traceId === undefined || log.trace_id === filter.traceId), + filter, + log => log.timestamp, + log => log.id, + ), + ); + }, + getProfile: id => getSnapshot().profiles.get(id), + getProfileFunctionSummaries(filter = {}) { + validate(filter); + return read("profiles", filter, snapshot => + readonlyCopy( + named( + getProfileFunctionSummaries( + new Map( + Array.from(snapshot.profiles).filter( + ([id, profile]) => + (filter.traceId === undefined || id === filter.traceId) && within(profile.timestamp, filter), + ), + ) as unknown as Parameters[0], + ), + filter, + summary => summary.name, + ), + ), + ); + }, + getQuerySummaries: ((filter: DatabaseReadFilter = {}) => { + validate(filter); + if (filter.description !== undefined && typeof filter.description !== "string") + throw new RangeError("description must be a string"); + if (filter.includeSpans !== undefined && typeof filter.includeSpans !== "boolean") + throw new RangeError("includeSpans must be boolean"); + return read("queries", filter, snapshot => + readonlyCopy( + named( + getQuerySummaries( + ordered( + spans(snapshot, filter), + { fromMs: filter.fromMs, toMs: filter.toMs, order: filter.order }, + span => span.timestamp, + span => span.span_id, + ) as Span[], + { description: filter.description }, + ).map(summary => { + const { spans: detail, ...compact } = summary; + return filter.includeSpans ? { ...compact, spans: detail } : compact; + }), + filter, + summary => summary.description, + ), + ), + ); + }) as TelemetryReader["getQuerySummaries"], + getResourceSummaries: ((filter: SummaryReadFilter = {}) => { + validate(filter); + if (filter.includeSpans !== undefined && typeof filter.includeSpans !== "boolean") + throw new RangeError("includeSpans must be boolean"); + return read("resources", filter, snapshot => + readonlyCopy( + named( + getResourceSummaries( + ordered( + spans(snapshot, filter), + { fromMs: filter.fromMs, toMs: filter.toMs, order: filter.order }, + span => span.timestamp, + span => span.span_id, + ) as Span[], + ).map(summary => { + const { spans: detail, ...compact } = summary; + return filter.includeSpans ? { ...compact, spans: detail } : compact; + }), + filter, + summary => summary.description, + ), + ), + ); + }) as TelemetryReader["getResourceSummaries"], + getAITraces(filter = {}) { + validate(filter); + return read("ai", filter, snapshot => + readonlyCopy( + ordered( + getAITraces(traces(snapshot, filter) as Trace[]), + filter, + trace => trace.timestamp, + trace => trace.id, + ), + ), + ); + }, + getWebVitals(filter = {}) { + validate(filter); + return read("vitals", filter, snapshot => + readonlyCopy( + ordered( + getWebVitals(events(snapshot, filter) as SentryEvent[]), + filter, + event => event.timestamp, + event => event.event_id, + ), + ), + ); + }, + getSdks: () => getSnapshot().sdks, + readPage(collection, options = {}) { + const invalid = (code: "invalid_query" | "invalid_cursor" | "stale_cursor", message: string) => + Object.freeze({ status: "invalid" as const, revision: getSnapshot().revision, code, message }); + try { + validate(options, true); + if (!["events", "traces", "logs", "spans"].includes(collection)) throw new RangeError("Unknown collection"); + if (collection !== "events" && options.spanId !== undefined) + throw new RangeError("spanId is only supported for events"); + } catch (error) { + return invalid("invalid_query", error instanceof Error ? error.message : "Invalid query"); + } + const { cursor, ...filter } = options; + const query = canonical({ ...filter, order: filter.order ?? "asc", limit: filter.limit ?? 100 }); + const snapshot = getSnapshot(); + let offset = 0; + if (cursor !== undefined) { + try { + if (typeof cursor !== "string" || cursor.length > 16384) throw new Error(); + const parsed = JSON.parse(decodeURIComponent(cursor)); + if ( + parsed.version !== 1 || + !Number.isSafeInteger(parsed.revision) || + !Number.isSafeInteger(parsed.offset) || + parsed.offset < 1 || + parsed.collection !== collection || + parsed.query !== query + ) + throw new Error(); + if (parsed.revision !== snapshot.revision) + return invalid("stale_cursor", "The snapshot changed; restart pagination"); + offset = parsed.offset; + } catch { + return invalid("invalid_cursor", "Cursor does not match this query"); + } + } + return read(`page:${collection}`, { query, cursor }, () => { + const { limit: _limit, ...allFilter } = filter; + const items = + collection === "events" + ? reader.getEvents(allFilter) + : collection === "traces" + ? reader.getTraces(allFilter) + : collection === "logs" + ? reader.getLogs(allFilter) + : reader.getSpans(allFilter); + if (offset > items.length || offset % (filter.limit ?? 100) !== 0) + return invalid("invalid_cursor", "Cursor offset is outside this query"); + const end = offset + (filter.limit ?? 100); + return Object.freeze({ + status: "ok" as const, + revision: snapshot.revision, + items: Object.freeze(items.slice(offset, end)), + total: items.length, + ...(end < items.length + ? { + nextCursor: encodeURIComponent( + JSON.stringify({ version: 1, revision: snapshot.revision, collection, query, offset: end }), + ), + } + : {}), + }); + }) as ReadPageResult>; + }, + }; + return Object.freeze(reader); +} diff --git a/packages/spotlight/src/telemetry/serialization.spec.ts b/packages/spotlight/src/telemetry/serialization.spec.ts new file mode 100644 index 000000000..3b8d0cbd9 --- /dev/null +++ b/packages/spotlight/src/telemetry/serialization.spec.ts @@ -0,0 +1,214 @@ +// @vitest-environment node +import { expect, test } from "vitest"; +import { SDK_CAPABILITIES } from "./capabilities.ts"; +import { serializeSnapshot, serializeTrace } from "./serialization.ts"; +import type { Span, Trace } from "./types.ts"; +function cyclicTrace(): Trace { + const parent: Span = { + span_id: "parent", + start_timestamp: 1, + timestamp: 3, + data: { flags: new Set(["one", "two"]), entries: new Map([["key", "value"]]) }, + }; + const child: Span = { span_id: "child", start_timestamp: 2, timestamp: 3 }; + parent.children = [child]; + child.children = [parent]; + return { + trace_id: "trace", + transactions: [], + errors: 0, + start_timestamp: 1, + timestamp: 3, + rootTransaction: null, + rootTransactionName: "", + spans: new Map([["parent", parent]]), + spanTree: [parent], + profileGrafted: false, + }; +} +test("trace JSON retains cyclic children once with child IDs", () => { + const dto = JSON.parse(JSON.stringify(serializeTrace(cyclicTrace()))); + expect(dto.schemaVersion).toBe(1); + expect(dto.spanTree).toEqual(["parent"]); + expect(dto.spans.map(([id]: [string]) => id)).toEqual(["parent", "child"]); + expect(dto.spans[0][1]).toMatchObject({ + span_id: "parent", + childSpanIds: ["child"], + data: { flags: ["one", "two"], entries: [["key", "value"]] }, + }); + expect(dto.spans[1][1].childSpanIds).toEqual(["parent"]); + expect(dto.spans[0][1]).not.toHaveProperty("children"); +}); +test("snapshot round trips plain JSON maps and profile data", () => { + const dto = serializeSnapshot({ + revision: 4, + envelopes: [], + events: [], + traces: [cyclicTrace()], + logs: [], + sdks: [], + profiles: new Map([["trace", { timestamp: 3, active_thread_id: "main", samples: [], stacks: [], frames: [] }]]), + }); + expect(JSON.parse(JSON.stringify(dto))).toEqual(dto); + expect(dto).toMatchObject({ schemaVersion: 1, revision: 4, profiles: [["trace", { active_thread_id: "main" }]] }); +}); +test("capabilities expose immutable supported formats", () => { + expect(SDK_CAPABILITIES).toMatchObject({ + schemaVersion: 1, + spanFormats: ["legacy", "v2"], + profileFormats: ["v1", "v2"], + }); + expect(() => (SDK_CAPABILITIES.itemTypes as unknown as string[]).push("future")).toThrow(); +}); + +test("span tree and transaction copies merge their child references without dropping descendants", () => { + const trace = cyclicTrace(); + const parent = trace.spans.get("parent")!; + const transaction = { + event_id: "transaction", + type: "transaction" as const, + timestamp: 3, + start_timestamp: 1, + contexts: { trace: { trace_id: "trace", span_id: "root" } }, + spans: [{ ...parent, children: undefined }], + }; + trace.transactions.push(transaction); + parent.transaction = transaction; + const dto = serializeTrace(trace); + expect(dto.spans).toHaveLength(2); + expect(dto.spans[0][1]).toMatchObject({ childSpanIds: ["child"], transactionEventId: "transaction" }); + expect(dto.transactions).toMatchObject([{ event_id: "transaction", spanIds: ["parent"] }]); +}); + +test("valid continuous profiles and streamed plus legacy traces serialize", async () => { + const { readFileSync } = await import("node:fs"); + const { createPipeline } = await import("../sdk/node.ts"); + const pipeline = createPipeline(); + try { + for (const file of [ + "continuous_profiling/transaction.txt", + "continuous_profiling/profile_chunk_1.txt", + "envelope_span_v2.txt", + ]) { + const receipt = await pipeline.ingestPayload({ + body: readFileSync(new URL(`../../_fixtures/${file}`, import.meta.url)), + contentType: "application/x-sentry-envelope", + }); + expect(receipt.status).toBe("processed"); + } + const dto = serializeSnapshot(pipeline.getSnapshot()); + expect(JSON.parse(JSON.stringify(dto))).toEqual(dto); + expect(dto.traces.length).toBeGreaterThan(0); + for (const trace of dto.traces) expect(trace.spans.length).toBeGreaterThan(0); + } finally { + pipeline.dispose(); + } +}); + +test("a wide span graph does not use call-argument expansion", () => { + const trace = cyclicTrace(); + const parent = trace.spans.get("parent")!; + parent.children = Array.from({ length: 130000 }, (_, index) => ({ + span_id: `child-${index}`, + start_timestamp: 2, + timestamp: 3, + })); + const dto = serializeTrace(trace); + expect(dto.spans).toHaveLength(130001); + expect(dto.spans[0][1].childSpanIds).toHaveLength(130000); +}); + +test("canonical spans serialize after streamed updates supersede transaction span copies", async () => { + const { createIngester } = await import("./index.ts"); + const client = createIngester(); + try { + expect( + ( + await client.ingestEnvelope([ + {}, + [ + [ + { type: "transaction" }, + { + event_id: "old", + type: "transaction", + start_timestamp: 1, + timestamp: 2, + contexts: { trace: { trace_id: "mixed", span_id: "root" } }, + spans: [ + { + span_id: "child", + parent_span_id: "root", + start_timestamp: 1.1, + timestamp: 1.2, + description: "old", + }, + ], + }, + ], + ], + ]) + ).status, + ).toBe("processed"); + expect( + ( + await client.ingestEnvelope([ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "mixed", + span_id: "child", + parent_span_id: "root", + start_timestamp: 1.1, + end_timestamp: 1.3, + name: "updated", + }, + ], + }, + ], + ], + ]) + ).status, + ).toBe("processed"); + const dto = serializeTrace(client.getTrace("mixed")!); + expect(dto.spans.find(([id]) => id === "child")?.[1]).toMatchObject({ description: "updated", timestamp: 1300 }); + } finally { + client.dispose(); + } +}); + +test("summary JSON converts profile trace ID sets to arrays", async () => { + const { getProfileFunctionSummaries } = await import("./selectors/profiles.ts"); + const { serializeSummaries } = await import("./serialization.ts"); + const summaries = getProfileFunctionSummaries( + new Map([ + [ + "trace-one", + { + samples: [ + { start_timestamp: 0, stack_id: 0, thread_id: "main" }, + { start_timestamp: 10, stack_id: 0, thread_id: "main" }, + ], + stacks: [[0]], + frames: [{ function: "work" }], + }, + ], + ]), + ); + const dto = serializeSummaries(summaries); + expect(dto).toEqual({ + schemaVersion: 1, + summaries: [{ name: "@work", totalDurationMs: 10, sampleCount: 1, traceIds: ["trace-one"] }], + }); + expect(JSON.parse(JSON.stringify(dto))).toEqual(dto); + expect(() => + serializeSummaries([ + { description: "query", count: 1, totalDurationMs: 1, averageDurationMs: 1, spans: [] }, + ] as never), + ).toThrow(TypeError); +}); diff --git a/packages/spotlight/src/telemetry/serialization.ts b/packages/spotlight/src/telemetry/serialization.ts new file mode 100644 index 000000000..971aa30ca --- /dev/null +++ b/packages/spotlight/src/telemetry/serialization.ts @@ -0,0 +1,150 @@ +import type { + DeepReadonly, + IngesterSnapshot, + ProcessedSpan, + ProcessedTrace, + ProfileFunctionSummary, +} from "./public.ts"; +import type { CompactQuerySummary, CompactResourceSummary } from "./reader.ts"; + +export type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; +export type SerializedSpan = { [key: string]: JsonValue; span_id: string; childSpanIds: string[] }; +export type SerializedTrace = { + [key: string]: JsonValue; + schemaVersion: 1; + trace_id: string; + spans: [string, SerializedSpan][]; + spanTree: string[]; +}; +export type SerializedSnapshot = { + schemaVersion: 1; + revision: number; + envelopes: JsonValue[]; + events: JsonValue[]; + traces: SerializedTrace[]; + logs: JsonValue[]; + sdks: JsonValue[]; + profiles: [string, JsonValue][]; +}; +export type SerializableSummary = + | ((CompactQuerySummary | CompactResourceSummary) & { readonly spans?: never }) + | ProfileFunctionSummary; +export type SerializedSummaries = { schemaVersion: 1; summaries: JsonValue[] }; + +/** Explicitly reject unsupported data rather than quietly replacing graph cycles with null. */ +function plain(value: unknown, ancestors = new Set()): JsonValue { + if (value === null) return null; + if (typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new TypeError("Telemetry JSON requires finite numbers"); + return value; + } + if (typeof value !== "object") throw new TypeError("Unsupported telemetry JSON value"); + if (ancestors.has(value)) throw new TypeError("Cycle outside span children cannot be serialized"); + ancestors.add(value); + try { + if (value instanceof Map) + return Array.from(value, ([key, item]) => [plain(key, ancestors), plain(item, ancestors)]); + if (value instanceof Set) return Array.from(value, item => plain(item, ancestors)); + if (value instanceof Uint8Array) return Array.from(value); + if (Array.isArray(value)) return value.map(item => (item === undefined ? null : plain(item, ancestors))); + const record = value as Record; + const result: { [key: string]: JsonValue } = {}; + const isSpan = + typeof record.span_id === "string" && + typeof record.start_timestamp === "number" && + typeof record.timestamp === "number"; + for (const [key, item] of Object.entries(record).sort(([a], [b]) => a.localeCompare(b))) { + if (item === undefined || (isSpan && (key === "children" || key === "transaction"))) continue; + Object.defineProperty(result, key, { + value: plain(item, ancestors), + enumerable: true, + writable: true, + configurable: true, + }); + } + if (isSpan) { + result.childSpanIds = ((record.children ?? []) as readonly ProcessedSpan[]).map(child => child.span_id); + if (record.transaction) result.transactionEventId = (record.transaction as { event_id: string }).event_id; + } + return result; + } finally { + ancestors.delete(value); + } +} + +/** Flat span entries preserve child cycles and retain children missing from the trace's span map. */ +export function serializeTrace(trace: ProcessedTrace): SerializedTrace { + const queue: ProcessedSpan[] = [...trace.spans.values(), ...trace.spanTree]; + const canonicalIds = new Set(Array.from(trace.spans.values(), span => span.span_id)); + const transactionsSeen = new Set>(); + function enqueueTransaction(transaction: NonNullable) { + if (transactionsSeen.has(transaction)) return; + transactionsSeen.add(transaction); + for (const span of transaction.spans ?? []) queue.push(span); + } + for (const transaction of trace.transactions) enqueueTransaction(transaction); + if (trace.rootTransaction) enqueueTransaction(trace.rootTransaction); + const seen = new Set(); + const byId = new Map(); + for (let index = 0; index < queue.length; index++) { + const span = queue[index]; + if (seen.has(span)) continue; + seen.add(span); + const serialized = plain(span) as SerializedSpan; + const existing = byId.get(span.span_id); + // The trace map is authoritative over historical copies in transactions. + // Children found only in those copies still belong in the flat graph. + if (existing) { + if (!canonicalIds.has(span.span_id)) + for (const [key, value] of Object.entries(serialized)) { + if (key === "childSpanIds") continue; + if (Object.hasOwn(existing, key) && JSON.stringify(existing[key]) !== JSON.stringify(value)) + throw new TypeError(`Conflicting span ID: ${span.span_id}`); + Object.defineProperty(existing, key, { value, enumerable: true, writable: true, configurable: true }); + } + existing.childSpanIds = Array.from(new Set([...existing.childSpanIds, ...serialized.childSpanIds])); + } else byId.set(span.span_id, serialized); + for (const child of span.children ?? []) queue.push(child); + if (span.transaction) enqueueTransaction(span.transaction); + } + const { spans: _spans, spanTree: _tree, transactions, rootTransaction, ...metadata } = trace; + const transactionJson = (transaction: DeepReadonly>): JsonValue => { + const { spans, ...rest } = transaction; + return { ...(plain(rest) as { [key: string]: JsonValue }), spanIds: (spans ?? []).map(span => span.span_id) }; + }; + return { + ...(plain(metadata) as { [key: string]: JsonValue }), + schemaVersion: 1, + trace_id: trace.trace_id, + transactions: transactions.map(transactionJson), + rootTransaction: rootTransaction ? transactionJson(rootTransaction) : null, + spans: Array.from(byId), + spanTree: trace.spanTree.map(span => span.span_id), + }; +} + +/** A JSON-ready DTO: Map entries and Set values become arrays; no custom JSON replacer is needed. */ +export function serializeSnapshot(snapshot: IngesterSnapshot): SerializedSnapshot { + return { + schemaVersion: 1, + revision: snapshot.revision, + envelopes: snapshot.envelopes.map(envelope => plain(envelope)), + events: snapshot.events.map(event => plain(event)), + traces: snapshot.traces.map(serializeTrace), + logs: snapshot.logs.map(log => plain(log)), + sdks: snapshot.sdks.map(sdk => plain(sdk)), + profiles: Array.from(snapshot.profiles, ([id, profile]) => [id, plain(profile)]), + }; +} + +/** Serialize compact read results, including profile trace ID sets, without losing a detailed span graph. */ +export function serializeSummaries(summaries: readonly SerializableSummary[]): SerializedSummaries { + return { + schemaVersion: 1, + summaries: summaries.map(summary => { + if ("spans" in summary) throw new TypeError("Use compact summaries or serializeTrace for detailed span graphs"); + return plain(summary); + }), + }; +} diff --git a/packages/spotlight/src/ui/telemetry/store/store.ts b/packages/spotlight/src/ui/telemetry/store/store.ts index af6b99432..2d0651d9c 100644 --- a/packages/spotlight/src/ui/telemetry/store/store.ts +++ b/packages/spotlight/src/ui/telemetry/store/store.ts @@ -13,24 +13,20 @@ export function setTelemetrySidecarUrl(url: string) { } const spotlight = createIngester({ - enrichStacktrace: async event => { + enrichStacktrace: async (event, { signal }) => { await Promise.all( (event.exception?.values ?? []).map(async exception => { const frames = exception.stacktrace?.frames; if (!frames || frames.every(frame => frame.post_context && frame.pre_context && frame.context_line)) return; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 5000); try { const response = await getNativeFetchImplementation()(contextLinesProvider, { method: "PUT", body: JSON.stringify(exception.stacktrace), - signal: controller.signal, + signal, }); if (response.status === 200) exception.stacktrace = await response.json(); } catch { // Optional context must not prevent sibling exceptions from finishing enrichment. - } finally { - clearTimeout(timeout); } }), ); diff --git a/packages/spotlight/tests/types/sdk-core.ts b/packages/spotlight/tests/types/sdk-core.ts index 674e540e2..210da8a43 100644 --- a/packages/spotlight/tests/types/sdk-core.ts +++ b/packages/spotlight/tests/types/sdk-core.ts @@ -1,7 +1,18 @@ -import { type ProcessedEvent, type QuerySummary, createIngester } from "@spotlightjs/spotlight/sdk/core"; +import { + type IngestResult, + type ProcessedEvent, + type QuerySummary, + SDK_CAPABILITIES, + type TelemetryReader, + createIngester, + createTelemetryReader, + serializeSnapshot, + serializeSummaries, + serializeTrace, +} from "@spotlightjs/spotlight/sdk/core"; const telemetry = createIngester(); -const queries: readonly QuerySummary[] = telemetry.getQuerySummaries({ traceId: "trace" }); +const queries: readonly QuerySummary[] = telemetry.getQuerySummaries({ traceId: "trace", includeSpans: true }); const events: readonly ProcessedEvent[] = telemetry.getEvents({ traceId: "trace", spanId: "span" }); const envelope = telemetry.getEnvelope("envelope"); if (envelope) { @@ -26,4 +37,62 @@ const normalizedTimestamp: number = events[0].timestamp; void normalizedTimestamp; void queries[0].totalDurationMs; void queries[0].averageDurationMs; + +const reader: TelemetryReader = createTelemetryReader(telemetry.getSnapshot); +// @ts-expect-error Read-only consumers cannot ingest. +reader.ingestEnvelope; +const compact = reader.getQuerySummaries(); +// @ts-expect-error Full spans require includeSpans: true. +const full: readonly QuerySummary[] = compact; +void full; +const page = reader.readPage("events", { fromMs: 0, toMs: 10000, order: "desc", limit: 20 }); +if (page.status === "ok") { + const events: readonly ProcessedEvent[] = page.items; + void events; + const cursor: string | undefined = page.nextCursor; + void cursor; +} else { + const code: "stale_cursor" | "invalid_cursor" | "invalid_query" = page.code; + void code; +} +const snapshot = serializeSnapshot(reader.getSnapshot()); +const version: 1 = snapshot.schemaVersion; +const unit: "unix-milliseconds" = SDK_CAPABILITIES.timestampUnit; +const trace = reader.getTrace("trace"); +if (trace) JSON.stringify(serializeTrace(trace)); +JSON.stringify(snapshot); +JSON.stringify(serializeSummaries(compact)); +JSON.stringify(serializeSummaries(reader.getProfileFunctionSummaries())); +// @ts-expect-error Export full span graphs through serializeTrace instead. +serializeSummaries(queries); +void version; +void unit; +async function receipts() { + const results: readonly IngestResult[] = await telemetry.ingestBatch([null], { + signal: new AbortController().signal, + }); + const receipt = await telemetry.ingestEnvelope(null, { deliveryId: "retry" }); + const retained: boolean = receipt.retained; + const revision: number = receipt.revision; + const duplicates: number = receipt.counts.duplicateEvents; + // @ts-expect-error Receipts are immutable. + receipt.counts.events = 0; + if (receipt.status === "ignored") { + // @ts-expect-error Reasons are a literal union. + const reason: "arbitrary-text" = receipt.reason; + void reason; + } + if (receipt.status === "invalid") { + const severity: "warning" | "error" = receipt.diagnostics[0].severity; + const index: number | undefined = receipt.diagnostics[0].itemIndex; + void severity; + void index; + } + await telemetry.drain(); + void results; + void retained; + void revision; + void duplicates; +} +void receipts; telemetry.dispose(); diff --git a/packages/spotlight/tests/types/sdk-node.ts b/packages/spotlight/tests/types/sdk-node.ts new file mode 100644 index 000000000..4605190f3 --- /dev/null +++ b/packages/spotlight/tests/types/sdk-node.ts @@ -0,0 +1,29 @@ +import type { IngestResult, TelemetryReader } from "@spotlightjs/spotlight/sdk/core"; +import { type PipelineInput, createPipeline } from "@spotlightjs/spotlight/sdk/node"; + +const pipeline = createPipeline({ + maxPending: 4, + maxPayloadBytes: 1024, + maxDecodedBytes: 2048, + maxRawBytes: 4096, + rawBufferSize: 3, + retention: { maxEnvelopes: 10, maxBytes: 8192, maxAgeMs: 1000 }, + enrichStacktrace: async (_event, { signal }) => { + signal.throwIfAborted(); + }, +}); +const reader: TelemetryReader = pipeline; +const input: PipelineInput = { body: new Uint8Array(), deliveryId: "request-1", signal: new AbortController().signal }; +const receipt: Promise = pipeline.ingestPayload(input); +const batch: Promise = pipeline.ingestPayloadBatch([input]); +const decoded = pipeline.getDecodedPayload("envelope"); +const body: Uint8Array | undefined = decoded?.body; +// @ts-expect-error Serialized payload ingestion has an explicit name. +pipeline.ingest; +// @ts-expect-error Decoded history has an explicit name. +pipeline.getRawPayload; +void reader; +void receipt; +void batch; +void body; +pipeline.dispose(); diff --git a/packages/spotlight/tests/types/tsconfig.sdk-node.json b/packages/spotlight/tests/types/tsconfig.sdk-node.json new file mode 100644 index 000000000..828ff9216 --- /dev/null +++ b/packages/spotlight/tests/types/tsconfig.sdk-node.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.sdk-core.json", + "compilerOptions": { "types": ["node"] }, + "files": ["sdk-node.ts"] +} diff --git a/scripts/check-sdk-types.mjs b/scripts/check-sdk-types.mjs new file mode 100644 index 000000000..b19b7193f --- /dev/null +++ b/scripts/check-sdk-types.mjs @@ -0,0 +1,41 @@ +import { spawnSync } from "node:child_process"; +import { access, mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; + +const packageDirectory = fileURLToPath(new URL("../packages/spotlight/", import.meta.url)); +const require = createRequire(import.meta.url); +const tsc = require.resolve("typescript/bin/tsc"); +for (const entry of ["core", "node"]) { + for (const extension of ["js", "d.ts"]) { + try { + await access(join(packageDirectory, "dist", "sdk", `${entry}.${extension}`)); + } catch { + throw new Error("Build the Spotlight Node and UI packages before running test:sdk-types"); + } + } +} + +function run(script, args = []) { + const result = spawnSync(process.execPath, [script, ...args], { cwd: packageDirectory, stdio: "inherit" }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`SDK check failed (${result.status ?? result.signal})`); +} + +// These resolve the published export map and emitted declarations, with no source aliases. +run(tsc, ["--project", "tests/types/tsconfig.sdk-core.json"]); +run(tsc, ["--project", "tests/types/tsconfig.sdk-node.json"]); +const temporary = await mkdtemp(join(tmpdir(), "spotlight-sdk-examples-")); +try { + run(tsc, ["--project", "examples/sdk/tsconfig.json", "--outDir", temporary]); + await writeFile(join(temporary, "package.json"), '{"type":"module"}\n'); + const scope = join(temporary, "node_modules", "@spotlightjs"); + await mkdir(scope, { recursive: true }); + await symlink(packageDirectory, join(scope, "spotlight"), "junction"); + run(join(temporary, "lifecycle.js")); + console.log("Built SDK declarations, consumer contracts, and executable examples passed."); +} finally { + await rm(temporary, { recursive: true, force: true }); +} From ce86422fea39d16ab2494cb3def06c79eef30029 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Tue, 27 Oct 2026 15:42:51 +0530 Subject: [PATCH 4/7] fix(sdk): Use explicit compression dispatch --- packages/spotlight/src/sdk/node.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/spotlight/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts index c55b6673f..3b7108139 100644 --- a/packages/spotlight/src/sdk/node.ts +++ b/packages/spotlight/src/sdk/node.ts @@ -38,11 +38,7 @@ export type Pipeline = Ingester & { }; const envelopeContentType = "application/x-sentry-envelope"; -const decompressors = new Map([ - ["gzip", gunzip], - ["deflate", inflate], - ["br", brotliDecompress], -]); +const compressedEncodings = new Set(["gzip", "deflate", "br"]); function limit(name: string, value: number, allowZero = false): number { if (!Number.isSafeInteger(value) || value < (allowZero ? 0 : 1)) { @@ -121,8 +117,7 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { } const initialCancellation = cancelled(); if (initialCancellation) return initialCancellation; - const decompress = encoding ? decompressors.get(encoding) : undefined; - if (decompress) { + if (encoding && encoding !== "identity") { const controller = new AbortController(); const abort = () => controller.abort(); decodeControllers.add(controller); @@ -131,10 +126,23 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { // zlib applies this bound while producing output, before a compressed bomb can expand in memory. const decodeOptions = { maxOutputLength: maxDecodedBytes, signal: controller.signal }; body = await new Promise((resolve, reject) => { - decompress(body, decodeOptions, (error, output) => { + const completed = (error: Error | null, output: Buffer) => { if (error) reject(error); else resolve(output); - }); + }; + switch (encoding) { + case "gzip": + gunzip(body, decodeOptions, completed); + break; + case "deflate": + inflate(body, decodeOptions, completed); + break; + case "br": + brotliDecompress(body, decodeOptions, completed); + break; + default: + reject(new Error("Unsupported content encoding")); + } }); } catch (error) { const cancellation = cancelled(); @@ -262,7 +270,7 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { return invalid("unsupported_encoding", "Content encoding must be a string"); } const encoding = input.encoding?.trim().toLowerCase(); - if (encoding && encoding !== "identity" && !decompressors.has(encoding)) { + if (encoding && encoding !== "identity" && !compressedEncodings.has(encoding)) { return invalid("unsupported_encoding", `Unsupported content encoding: ${encoding}`); } const size = typeof input.body === "string" ? Buffer.byteLength(input.body, "utf8") : input.body.byteLength; From 3954c2ea89c04b8eb833eb8bdd6465e890d69e41 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Tue, 27 Oct 2026 15:53:36 +0530 Subject: [PATCH 5/7] fix(sdk): Reject invalid envelope lengths in every decode mode --- .../spotlight/src/telemetry/envelope/codec.spec.ts | 9 +++++++++ packages/spotlight/src/telemetry/envelope/codec.ts | 10 ++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/spotlight/src/telemetry/envelope/codec.spec.ts b/packages/spotlight/src/telemetry/envelope/codec.spec.ts index 991911f59..913339881 100644 --- a/packages/spotlight/src/telemetry/envelope/codec.spec.ts +++ b/packages/spotlight/src/telemetry/envelope/codec.spec.ts @@ -50,6 +50,15 @@ describe("portable byte envelope codec", () => { expect(errors).toHaveBeenCalledOnce(); }); + test.each([-1, -100, 1.5, "1", null, false])("rejects invalid item length %j in both decoder modes", length => { + const bytes = encode(`{}\n${JSON.stringify({ type: "attachment", length })}\n`); + for (const strict of [false, true]) { + const errors = vi.fn(); + expect(decodeEnvelope(bytes, undefined, errors, strict)).toBeNull(); + expect(errors).toHaveBeenCalledOnce(); + } + }); + test.each([ ["sentry.javascript.nextjs", "Mozilla/5.0 Chrome/131.0.0.0 Safari/537.36", "browser"], ["sentry.cocoa", "Mozilla/5.0 Safari/605", "mobile"], diff --git a/packages/spotlight/src/telemetry/envelope/codec.ts b/packages/spotlight/src/telemetry/envelope/codec.ts index 8e5aa2215..e50ddec5d 100644 --- a/packages/spotlight/src/telemetry/envelope/codec.ts +++ b/packages/spotlight/src/telemetry/envelope/codec.ts @@ -64,13 +64,15 @@ export function decodeEnvelope( return null; } const payloadLength = itemHeader.length; + // Permissive mode allows truncated payloads, never invalid buffer indices. + if (payloadLength !== undefined && (!Number.isSafeInteger(payloadLength) || payloadLength < 0)) { + reportError("Malformed envelope item length, skipping envelope..."); + return null; + } if ( strict && payloadLength !== undefined && - (!Number.isInteger(payloadLength) || - payloadLength < 0 || - payloadLength > buffer.length || - (payloadLength < buffer.length && buffer[payloadLength] !== 0xa)) + (payloadLength > buffer.length || (payloadLength < buffer.length && buffer[payloadLength] !== 0xa)) ) { reportError("Malformed envelope item length, skipping envelope..."); return null; From 0bc19e63dfbd7fe8763ad842879f07ef7e55f141 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Wed, 28 Oct 2026 15:11:50 +0530 Subject: [PATCH 6/7] feat(sdk): Clarify telemetry APIs and support live retention Use telemetry store and pipeline names, explicit envelope and payload batch methods, and decoded history options. Preserve live retention and immutable snapshots for deeply nested span graphs. Co-Authored-By: Codex --- README.md | 2 +- packages/spotlight/README.md | 30 +-- packages/spotlight/examples/sdk/frontend.ts | 4 +- packages/spotlight/examples/sdk/lifecycle.ts | 12 +- packages/spotlight/examples/sdk/react.tsx | 6 +- packages/spotlight/examples/sdk/reader.ts | 4 +- packages/spotlight/examples/sdk/server.ts | 8 +- packages/spotlight/src/sdk/core.ts | 10 +- packages/spotlight/src/sdk/node.spec.ts | 163 +++++++------- packages/spotlight/src/sdk/node.ts | 114 +++++----- packages/spotlight/src/server/sdk.spec.ts | 56 ++--- .../src/telemetry/attachments.spec.ts | 8 +- .../spotlight/src/telemetry/client.spec.ts | 46 ++-- packages/spotlight/src/telemetry/index.ts | 55 +++-- .../src/telemetry/internal/readonly.spec.ts | 26 +++ .../src/telemetry/internal/readonly.ts | 33 +-- .../src/telemetry/internal/retention.ts | 16 ++ .../spotlight/src/telemetry/lifecycle.spec.ts | 200 ++++++++++++++++-- .../spotlight/src/telemetry/parity.spec.ts | 20 +- packages/spotlight/src/telemetry/public.ts | 16 +- .../spotlight/src/telemetry/reader.spec.ts | 16 +- packages/spotlight/src/telemetry/reader.ts | 32 +-- .../src/telemetry/selectors/database.ts | 2 +- .../src/telemetry/selectors/index.ts | 2 +- .../src/telemetry/selectors/selectors.test.ts | 10 +- .../src/telemetry/serialization.spec.ts | 8 +- .../spotlight/src/telemetry/serialization.ts | 4 +- .../components/events/EventContexts.test.tsx | 4 +- .../telemetry/components/insights/Queries.tsx | 6 +- .../components/insights/QuerySummary.tsx | 4 +- .../spotlight/src/ui/telemetry/store/store.ts | 10 +- packages/spotlight/tests/types/sdk-core.ts | 12 +- packages/spotlight/tests/types/sdk-node.ts | 17 +- 33 files changed, 608 insertions(+), 348 deletions(-) create mode 100644 packages/spotlight/src/telemetry/internal/readonly.spec.ts create mode 100644 packages/spotlight/src/telemetry/internal/retention.ts diff --git a/README.md b/README.md index bfba4bd5f..f7bbb36ab 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Spotlight uses Sentry SDKs to capture telemetry data and sends it to the Sidecar ## Embed the telemetry SDK -Use `createIngester()` from `@spotlightjs/spotlight/sdk/core` to process parsed Sentry envelopes in portable runtimes, or `createPipeline()` from `@spotlightjs/spotlight/sdk/node` for serialized and compressed Node payloads. Both share the same processing, receipts, readers, and retention controls. +Use `createTelemetryStore()` from `@spotlightjs/spotlight/sdk/core` to process parsed Sentry envelopes in portable runtimes, or `createTelemetryPipeline()` from `@spotlightjs/spotlight/sdk/node` for serialized and compressed Node payloads. Both share the same processing, receipts, readers, and retention controls. See the [SDK reference](packages/spotlight/README.md) and [executable examples](packages/spotlight/examples/sdk) for integration, retries, cancellation, paging, JSON exports, and React subscriptions. diff --git a/packages/spotlight/README.md b/packages/spotlight/README.md index b63ccb23e..80a079d15 100644 --- a/packages/spotlight/README.md +++ b/packages/spotlight/README.md @@ -10,8 +10,8 @@ comparing their retained histories. | Entry | Factory | Input | | --- | --- | --- | -| `@spotlightjs/spotlight/sdk/core` | `createIngester()` | Parsed Sentry envelopes in portable runtimes | -| `@spotlightjs/spotlight/sdk/node` | `createPipeline()` | Adds serialized bodies, decompression, and decoded payload history | +| `@spotlightjs/spotlight/sdk/core` | `createTelemetryStore()` | Parsed Sentry envelopes in portable runtimes | +| `@spotlightjs/spotlight/sdk/node` | `createTelemetryPipeline()` | Adds serialized bodies, decompression, and decoded payload history | | `@spotlightjs/spotlight/sdk` | Existing buffer helpers | Compatibility for existing transport consumers | ## Ingest and read telemetry @@ -19,9 +19,9 @@ comparing their retained histories. Use `@spotlightjs/spotlight/sdk/core` for parsed Sentry envelopes in a browser, worker, or Node application. Use `@spotlightjs/spotlight/sdk/node` for serialized or compressed payloads on Node 20+. Both expose the same processing and read APIs; creating an instance does not start a server. The portable entry has no Node or React dependency. The existing raw-buffer helpers at `/sdk` remain available. ```ts -import { createIngester, serializeSnapshot } from "@spotlightjs/spotlight/sdk/core"; +import { createTelemetryStore, serializeSnapshot } from "@spotlightjs/spotlight/sdk/core"; -const telemetry = createIngester(); +const telemetry = createTelemetryStore(); const receipt = await telemetry.ingestEnvelope(parsedEnvelope, { deliveryId: "transport-request-123", // Reuse for retries of this delivery. signal: abortController.signal, @@ -38,16 +38,16 @@ const json = JSON.stringify(serializeSnapshot(telemetry.getSnapshot())); telemetry.dispose(); ``` -The Node adapter accepts `string`, `Uint8Array`, and `Buffer` bodies. `ingestPayload({ body, contentType?, encoding?, userAgent?, deliveryId?, signal? })` decodes `application/x-sentry-envelope` (the default content type) and supports identity, gzip, deflate, and Brotli encoding. Use `getDecodedPayload(envelopeId)` for a copied decompressed payload when it remains in the independently bounded history. Parsed envelopes still use `ingestEnvelope`; the payload methods are `ingestPayload` and `ingestPayloadBatch`. +The Node adapter accepts `string`, `Uint8Array`, and `Buffer` bodies. `ingestPayload({ body, contentType?, encoding?, userAgent?, deliveryId?, signal? })` decodes `application/x-sentry-envelope` (the default content type) and supports identity, gzip, deflate, and Brotli encoding. Use `getDecodedPayload(envelopeId)` for a copied decompressed payload when it remains in the independently bounded history. Parsed envelopes still use `ingestEnvelope`; the payload methods are `ingestPayload` and `ingestPayloads`. ```ts -import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; +import { createTelemetryPipeline } from "@spotlightjs/spotlight/sdk/node"; -const pipeline = createPipeline({ rawBufferSize: 100 }); +const pipeline = createTelemetryPipeline({ decodedHistory: { maxEntries: 100 } }); const receipt = await pipeline.ingestPayload({ body: envelopeBody, deliveryId: "request-1" }); if (receipt.status === "processed") { const payload = pipeline.getDecodedPayload(receipt.envelopeId); // Defensive decoded-byte copy. - const queries = pipeline.getQuerySummaries({ traceId }); // Compact by default. + const queries = pipeline.getDatabaseQuerySummaries({ traceId }); // Compact by default. } await pipeline.drain(); pipeline.dispose(); @@ -72,7 +72,7 @@ Branch on diagnostic `code` and `severity`, or ignored `reason`, rather than mes Delivery IDs deduplicate only while the corresponding envelope is retained. Eviction and `reset()` permit that delivery again. There is no content hashing: separately allocated envelopes without IDs are separate observations. Reusing the same parsed object or its explicit Spotlight envelope ID preserves its delivery identity within the retained window. Event-ID deduplication is separate from delivery deduplication. -`ingestBatch(iterable, { signal? })` returns receipts in input order and preserves valid neighbors of invalid inputs. It is not a transaction. `ingestPayloadBatch(inputs)` does the same for Node inputs; each input carries its own signal and delivery ID. A batch occupies one queued operation, including while active. Items are copied as they are reached rather than copying the entire batch up front: keep queued batch items unchanged until their receipts resolve. Individual `ingestEnvelope` and `ingestPayload` calls copy their input before the first await. Cancellation returns `ignored/aborted`. `drain()` waits for operations already submitted at the call, excluding future work. `getStats()` reports pending operations, retained envelope count/bytes, revision, and disposal. `subscribe((snapshot, change) => ...)` returns an unsubscribe function; `change` identifies ingest, eviction, reset, or disposal and affected envelope IDs. Release timers and listeners with `dispose()`. +`ingestEnvelopes(iterable, { signal? })` returns receipts in input order and preserves valid neighbors of invalid inputs. It is not a transaction. `ingestPayloads(inputs)` does the same for Node inputs; each input carries its own signal and delivery ID. A batch occupies one queued operation, including while active. Items are copied as they are reached rather than copying the entire batch up front: keep queued batch items unchanged until their receipts resolve. Individual `ingestEnvelope` and `ingestPayload` calls copy their input before the first await. Cancellation returns `ignored/aborted`. `drain()` waits for operations already submitted at the call, excluding future work. `getStats()` reports pending operations, retained envelope count/bytes, revision, and disposal. `subscribe((snapshot, change) => ...)` returns an unsubscribe function; `change` identifies ingest, eviction, reset, or disposal and affected envelope IDs. Release timers and listeners with `dispose()`. An optional `enrichStacktrace(event, { signal })` hook receives a copied event. Observe its signal to cancel external work. A timeout reports a warning and keeps telemetry; abort, reset, disposal, or timeout prevent late hook writes from entering published state. @@ -81,8 +81,8 @@ An optional `enrichStacktrace(event, { signal })` hook receives a copied event. Accept a `TelemetryReader` from `/sdk/core` in consumers that only query telemetry. Both factories implement it. `createTelemetryReader(getSnapshot)` also constructs a reader over an existing snapshot source. Singular getters such as `getTrace(id)` and `getEvent(id)` return `undefined` when absent; collection getters return readonly arrays. All processed timestamps and durations use milliseconds; time bounds are inclusive Unix milliseconds. ```ts -const queries = reader.getQuerySummaries({ traceId, fromMs, toMs, limit: 10 }); -const detailed = reader.getQuerySummaries({ traceId, includeSpans: true }); +const queries = reader.getDatabaseQuerySummaries({ traceId, fromMs, toMs, limit: 10 }); +const detailed = reader.getDatabaseQuerySummaries({ traceId, includeSpans: true }); const first = reader.readPage("events", { traceId, limit: 100, order: "desc" }); if (first.status === "ok" && first.nextCursor) { const next = reader.readPage("events", { @@ -110,9 +110,11 @@ Use `serializeSummaries(reader.getProfileFunctionSummaries())` or pass compact q | `retention.maxEnvelopes` / `retention.maxBytes` | 500 / 64 MiB | | `retention.maxAgeMs` | No age limit; when configured, expires automatically since receipt | | Node `maxPayloadBytes` / `maxDecodedBytes` | 8 MiB / 32 MiB | -| Node `rawBufferSize` / `maxRawBytes` | 500 / 32 MiB | +| Node `decodedHistory.maxEntries` / `decodedHistory.maxBytes` | 500 / 32 MiB | -Limits must be positive safe integers; Node `rawBufferSize` and `maxRawBytes` also allow zero to disable decoded history. Byte limits use bytes, with 1 MiB = 1024 × 1024 bytes. The Node decoder bounds decompression output. Bound transport collection separately before passing a complete body to the SDK, as shown in the HTTP example. Retention evicts whole envelopes and the telemetry derived from them; the byte budget measures the serialized retained envelopes, not total JavaScript heap size. +`setRetention({ maxEnvelopes?, maxBytes?, maxAgeMs? })` updates only supplied retention limits and resolves after queued eviction. Existing subscriptions and Node decoded history remain available. Age limits cannot be cleared; disposed instances ignore updates. Node updates follow earlier payload/parsed work, and `drain()` includes retention updates submitted before the drain. + +Limits must be positive safe integers; Node `decodedHistory.maxEntries` and `decodedHistory.maxBytes` also allow zero to disable decoded history. Byte limits use bytes, with 1 MiB = 1024 × 1024 bytes. The Node decoder bounds decompression output. Bound transport collection separately before passing a complete body to the SDK, as shown in the HTTP example. Retention evicts whole envelopes and the telemetry derived from them; the byte budget measures the serialized retained envelopes, not total JavaScript heap size. `SDK_CAPABILITIES` exposes readonly machine-readable metadata for legacy and v2 spans, profile v1/v2, supported item types, and timestamp units. Supported item types include event/error, transaction, span, log, profile/profile_chunk, feedback, and attachment; individual wire versions can still be unsupported. Portable runtimes need standard JavaScript APIs including `AbortController`, `TextEncoder`, timers, maps, and sets. React is required only by your React consumer; compressed payload decoding is provided by the Node entry. @@ -126,7 +128,7 @@ The [SDK examples](examples/sdk) cover a bounded HTTP server, a frontend transpo | `getEvent(eventId)`, `getEvents(filter?)`, `getErrors(filter?)` | Normalized events and captured exceptions | | `getSpans(filter?)` | Normalized spans | | `getLog(logId)`, `getLogs(filter?)` | Logs and trace correlation | -| `getQuerySummaries(filter?)` | Database spans grouped by trimmed description | +| `getDatabaseQuerySummaries(filter?)` | Database spans grouped by trimmed description | | `getResourceSummaries(filter?)` | Resource timings and encoded sizes | | `getProfile(traceId)`, `getProfileFunctionSummaries(filter?)` | Associated profiles and inclusive frame durations/sample counts | | `getAITraces(filter?)` | AI interactions, tokens, prompts, responses, and tools | diff --git a/packages/spotlight/examples/sdk/frontend.ts b/packages/spotlight/examples/sdk/frontend.ts index e516b035f..b2ba9d39d 100644 --- a/packages/spotlight/examples/sdk/frontend.ts +++ b/packages/spotlight/examples/sdk/frontend.ts @@ -1,8 +1,8 @@ -import { type IngestOptions, createIngester } from "@spotlightjs/spotlight/sdk/core"; +import { type IngestOptions, createTelemetryStore } from "@spotlightjs/spotlight/sdk/core"; /** Keep one instance for the lifetime of the telemetry view. */ export function createFrontendTelemetry() { - const ingester = createIngester({ retention: { maxEnvelopes: 500 } }); + const ingester = createTelemetryStore({ retention: { maxEnvelopes: 500 } }); return { ingester, // Pass the parsed envelopes received over SSE, WebSocket, or your own transport. diff --git a/packages/spotlight/examples/sdk/lifecycle.ts b/packages/spotlight/examples/sdk/lifecycle.ts index 6c06106af..50366af29 100644 --- a/packages/spotlight/examples/sdk/lifecycle.ts +++ b/packages/spotlight/examples/sdk/lifecycle.ts @@ -1,10 +1,10 @@ -import { type RawEnvelope, SDK_CAPABILITIES } from "@spotlightjs/spotlight/sdk/core"; -import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; +import { SDK_CAPABILITIES, type SentryEnvelope } from "@spotlightjs/spotlight/sdk/core"; +import { createTelemetryPipeline } from "@spotlightjs/spotlight/sdk/node"; import { createFrontendTelemetry } from "./frontend.js"; import { inspectTrace, readEventIds } from "./reader.js"; // Parsed wire timestamps are seconds; normalized reader timestamps are Unix milliseconds. -function envelope(id: string): RawEnvelope { +function envelope(id: string): SentryEnvelope { return [ {}, [ @@ -32,7 +32,7 @@ function check(condition: unknown, message: string): asserts condition { } const { ingester, receiveEnvelope, disconnect } = createFrontendTelemetry(); -const pipeline = createPipeline(); +const pipeline = createTelemetryPipeline(); try { const input = envelope("one"); const first = await receiveEnvelope(input, { deliveryId: "request-1" }); @@ -41,7 +41,7 @@ try { check(retry.status === "ignored" && retry.reason === "duplicate", "Retained delivery retry"); // Each input gets a receipt. A malformed middle input does not roll back valid neighbors. - const batch = await ingester.ingestBatch([envelope("two"), null, envelope("three")]); + const batch = await ingester.ingestEnvelopes([envelope("two"), null, envelope("three")]); check( batch[0].status === "processed" && batch[1].status === "invalid" && batch[2].status === "processed", "Partial batch", @@ -69,7 +69,7 @@ try { check(readEventIds(ingester).status === "ok", "Reader-only pagination"); const wire = (id: string) => `{}\n${JSON.stringify({ type: "span" })}\n${JSON.stringify(envelope(id)[1][0][1])}`; - const payloads = await pipeline.ingestPayloadBatch([{ body: wire("node-one") }, { body: "invalid" }]); + const payloads = await pipeline.ingestPayloads([{ body: wire("node-one") }, { body: "invalid" }]); check(payloads[0].status === "processed" && payloads[1].status === "invalid", "Payload batch"); if (payloads[0].status === "processed") { check(pipeline.getDecodedPayload(payloads[0].envelopeId)?.body.length, "Copied decoded payload"); diff --git a/packages/spotlight/examples/sdk/react.tsx b/packages/spotlight/examples/sdk/react.tsx index 25c8cc4dd..138c3dfc9 100644 --- a/packages/spotlight/examples/sdk/react.tsx +++ b/packages/spotlight/examples/sdk/react.tsx @@ -1,16 +1,16 @@ -import type { Ingester, TelemetryReader } from "@spotlightjs/spotlight/sdk/core"; +import type { TelemetryReader, TelemetryStore } from "@spotlightjs/spotlight/sdk/core"; import { useSyncExternalStore } from "react"; type Props = { reader: TelemetryReader; - subscribe: Ingester["subscribe"]; + subscribe: TelemetryStore["subscribe"]; traceId?: string; }; /** Pass reader={ingester} subscribe={ingester.subscribe}; the view cannot ingest or reset. */ export function QueryList({ reader, subscribe, traceId }: Props) { useSyncExternalStore(subscribe, reader.getSnapshot, reader.getSnapshot); - const queries = reader.getQuerySummaries({ traceId }); + const queries = reader.getDatabaseQuerySummaries({ traceId }); return (
        {queries.map(query => ( diff --git a/packages/spotlight/examples/sdk/reader.ts b/packages/spotlight/examples/sdk/reader.ts index e48a48684..4b0e847d6 100644 --- a/packages/spotlight/examples/sdk/reader.ts +++ b/packages/spotlight/examples/sdk/reader.ts @@ -9,8 +9,8 @@ import { export function inspectTrace(reader: TelemetryReader, traceId: string) { const trace = reader.getTrace(traceId); return { - queries: reader.getQuerySummaries({ traceId, order: "desc", limit: 10 }), - detailedQueries: reader.getQuerySummaries({ traceId, includeSpans: true }), + queries: reader.getDatabaseQuerySummaries({ traceId, order: "desc", limit: 10 }), + detailedQueries: reader.getDatabaseQuerySummaries({ traceId, includeSpans: true }), traceJson: trace ? JSON.stringify(serializeTrace(trace)) : undefined, snapshotJson: JSON.stringify(serializeSnapshot(reader.getSnapshot())), profileJson: JSON.stringify(serializeSummaries(reader.getProfileFunctionSummaries({ traceId }))), diff --git a/packages/spotlight/examples/sdk/server.ts b/packages/spotlight/examples/sdk/server.ts index 610d9cf45..95f817b5f 100644 --- a/packages/spotlight/examples/sdk/server.ts +++ b/packages/spotlight/examples/sdk/server.ts @@ -1,13 +1,13 @@ import { createServer } from "node:http"; -import { createPipeline } from "@spotlightjs/spotlight/sdk/node"; +import { createTelemetryPipeline } from "@spotlightjs/spotlight/sdk/node"; const maxPayloadBytes = 8 * 1024 * 1024; /** Embed the SDK in an HTTP server; the SDK itself does not open a port. */ export function createTelemetryServer() { - const pipeline = createPipeline({ + const pipeline = createTelemetryPipeline({ maxPayloadBytes, - rawBufferSize: 100, + decodedHistory: { maxEntries: 100 }, retention: { maxEnvelopes: 1000, maxAgeMs: 10 * 60 * 1000 }, }); const server = createServer(async (request, response) => { @@ -46,7 +46,7 @@ export function createTelemetryServer() { } if (request.method === "GET" && url.pathname === "/queries") { const traceId = url.searchParams.get("traceId") ?? undefined; - response.end(JSON.stringify(pipeline.getQuerySummaries({ traceId }))); + response.end(JSON.stringify(pipeline.getDatabaseQuerySummaries({ traceId }))); return; } response.statusCode = 404; diff --git a/packages/spotlight/src/sdk/core.ts b/packages/spotlight/src/sdk/core.ts index bd04f8663..d72b1f9c7 100644 --- a/packages/spotlight/src/sdk/core.ts +++ b/packages/spotlight/src/sdk/core.ts @@ -1,9 +1,9 @@ -export { createIngester } from "../telemetry/index.ts"; +export { createTelemetryStore } from "../telemetry/index.ts"; export type { - Ingester, - IngesterOptions, - IngesterSnapshot, - RawEnvelope, + TelemetryStore, + TelemetryStoreOptions, + TelemetrySnapshot, + SentryEnvelope, ProcessedEvent, ProcessedSpan, ProcessedTrace, diff --git a/packages/spotlight/src/sdk/node.spec.ts b/packages/spotlight/src/sdk/node.spec.ts index 4ff407d0b..5bcd9f043 100644 --- a/packages/spotlight/src/sdk/node.spec.ts +++ b/packages/spotlight/src/sdk/node.spec.ts @@ -2,7 +2,7 @@ import { readFileSync } from "node:fs"; import { brotliCompressSync, deflateSync, gzipSync } from "node:zlib"; import { describe, expect, test, vi } from "vitest"; -import { createPipeline } from "./node.ts"; +import { createTelemetryPipeline } from "./node.ts"; const fixture = readFileSync(new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url)); @@ -16,7 +16,7 @@ describe("Node Telemetry adapter", () => { Buffer.from(`${JSON.stringify({ type, length: binary.length })}\n`), binary, ]); - const telemetry = createPipeline(); + const telemetry = createTelemetryPipeline(); const result = await telemetry.ingestPayload({ body }); expect(result.status).toBe("processed"); expect(telemetry.getEvent("with-opaque-item")).toBeDefined(); @@ -30,7 +30,7 @@ describe("Node Telemetry adapter", () => { test.each(['{}\n{"type":"attachment","length":0}', '{}\n{"type":"attachment"}'])( "rejects a truncated item header without its terminating newline: %s", async body => { - const telemetry = createPipeline(); + const telemetry = createTelemetryPipeline(); expect((await telemetry.ingestPayload({ body })).status).toBe("invalid"); expect(telemetry.getEnvelopes()).toEqual([]); telemetry.dispose(); @@ -38,7 +38,7 @@ describe("Node Telemetry adapter", () => { ); test("accepts an empty attachment payload after a complete item header", async () => { - const telemetry = createPipeline(); + const telemetry = createTelemetryPipeline(); expect((await telemetry.ingestPayload({ body: '{}\n{"type":"attachment","length":0}\n' })).status).toBe("ignored"); expect(telemetry.getEnvelopes()).toHaveLength(1); telemetry.dispose(); @@ -48,7 +48,7 @@ describe("Node Telemetry adapter", () => { ["processed", fixture], ["unsupported", Buffer.from('{}\n{"type":"attachment","length":5}\nhello')], ] as const)("subscribers observe %s raw bytes together with the envelope in one notification", async (_, body) => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const observations: (Buffer | undefined)[] = []; spotlight.subscribe(snapshot => { const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); @@ -59,7 +59,7 @@ describe("Node Telemetry adapter", () => { }); test("raw history bounds already apply when subscribers observe the next envelope", async () => { - const spotlight = createPipeline({ rawBufferSize: 1 }); + const spotlight = createTelemetryPipeline({ decodedHistory: { maxEntries: 1 } }); const first = await spotlight.ingestPayload({ body: fixture }); if (first.status !== "processed") throw new Error("Expected processed envelope"); const observations: { old: Buffer | undefined; newest: Buffer | undefined }[] = []; @@ -75,11 +75,11 @@ describe("Node Telemetry adapter", () => { }); test("defaults to Sentry envelopes and exposes processed raw payloads", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const result = await spotlight.ingestPayload({ body: fixture }); expect(result.status).toBe("processed"); if (result.status !== "processed") throw new Error("Expected processed envelope"); - expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); expect(spotlight.getDecodedPayload(result.envelopeId)).toEqual({ body: fixture, contentType: "application/x-sentry-envelope", @@ -92,25 +92,25 @@ describe("Node Telemetry adapter", () => { ["deflate", deflateSync], ["br", brotliCompressSync], ] as const)("decodes %s payloads", async (encoding, compress) => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const result = await spotlight.ingestPayload({ body: new Uint8Array(compress(fixture)), contentType: " Application/X-Sentry-Envelope ; charset=utf-8 ", encoding, }); expect(result.status).toBe("processed"); - expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); if (result.status !== "processed") throw new Error("Expected processed envelope"); expect(spotlight.getDecodedPayload(result.envelopeId)?.body).toEqual(fixture); }); test("accepts serialized envelope strings", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); expect((await spotlight.ingestPayload({ body: fixture.toString("utf8") })).status).toBe("processed"); }); test("returns an explicit ignored result for unsupported content types", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); expect(await spotlight.ingestPayload({ body: fixture, contentType: "application/json" })).toMatchObject({ status: "ignored", reason: expect.any(String), @@ -119,7 +119,7 @@ describe("Node Telemetry adapter", () => { }); test("retains raw payloads for valid envelopes containing only unsupported items", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const body = Buffer.from('{}\n{"type":"attachment","length":5}\nhello'); const result = await spotlight.ingestPayload({ body }); expect(result).toMatchObject({ status: "ignored", reason: "unsupported_items", envelopeId: expect.any(String) }); @@ -150,7 +150,7 @@ describe("Node Telemetry adapter", () => { const warnings = vi.spyOn(console, "warn").mockImplementation(() => {}); const logs = vi.spyOn(console, "log").mockImplementation(() => {}); try { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); // Runtime validation is part of the public boundary. const result = await spotlight.ingestPayload(input as Parameters[0]); expect(result).toMatchObject({ status: "invalid", diagnostics: [{ code }] }); @@ -164,7 +164,7 @@ describe("Node Telemetry adapter", () => { }); test("copies incoming and returned raw payload bytes", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const body = Buffer.from(fixture); const pending = spotlight.ingestPayload({ body }); body.fill(0); @@ -178,7 +178,7 @@ describe("Node Telemetry adapter", () => { }); test("bounds raw payload history independently from processed retention", async () => { - const spotlight = createPipeline({ rawBufferSize: 1 }); + const spotlight = createTelemetryPipeline({ decodedHistory: { maxEntries: 1 } }); const first = await spotlight.ingestPayload({ body: fixture }); const second = await spotlight.ingestPayload({ body: fixture }); if (first.status !== "processed" || second.status !== "processed") throw new Error("Expected envelopes"); @@ -191,7 +191,7 @@ describe("Node Telemetry adapter", () => { }); test("invalid ingestion cannot evict a retained raw payload", async () => { - const spotlight = createPipeline({ rawBufferSize: 1 }); + const spotlight = createTelemetryPipeline({ decodedHistory: { maxEntries: 1 } }); const first = await spotlight.ingestPayload({ body: fixture }); if (first.status !== "processed") throw new Error("Expected processed envelope"); expect((await spotlight.ingestPayload({ body: "not an envelope" })).status).toBe("invalid"); @@ -202,7 +202,7 @@ describe("Node Telemetry adapter", () => { }); test("retains at most 500 raw payloads by default", async () => { - const spotlight = createPipeline({ retention: { maxEnvelopes: 1 } }); + const spotlight = createTelemetryPipeline({ retention: { maxEnvelopes: 1 } }); const first = await spotlight.ingestPayload({ body: fixture }); let last = first; for (let i = 0; i < 500; i++) last = await spotlight.ingestPayload({ body: fixture }); @@ -213,16 +213,16 @@ describe("Node Telemetry adapter", () => { }); test("zero raw buffer capacity still processes telemetry", async () => { - const spotlight = createPipeline({ rawBufferSize: 0 }); + const spotlight = createTelemetryPipeline({ decodedHistory: { maxEntries: 0 } }); const result = await spotlight.ingestPayload({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); expect(spotlight.getDecodedPayload(result.envelopeId)).toBeUndefined(); - expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); }); test("processed age retention does not expire the separate raw history", async () => { vi.useFakeTimers(); - const spotlight = createPipeline({ retention: { maxAgeMs: 100 } }); + const spotlight = createTelemetryPipeline({ retention: { maxAgeMs: 100 } }); try { const result = await spotlight.ingestPayload({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); @@ -244,7 +244,7 @@ describe("Node Telemetry adapter", () => { const gate = new Promise(resolve => { release = resolve; }); - const spotlight = createPipeline({ + const spotlight = createTelemetryPipeline({ enrichStacktrace: async () => { start(); await gate; @@ -273,7 +273,7 @@ describe("Node Telemetry adapter", () => { test("delivers invalid-input diagnostics to the configured observer", async () => { const diagnostics: string[] = []; - const spotlight = createPipeline({ + const spotlight = createTelemetryPipeline({ onDiagnostic: diagnostic => { diagnostics.push(diagnostic.code); }, @@ -284,7 +284,7 @@ describe("Node Telemetry adapter", () => { }); test("throwing diagnostic observers do not reject invalid input", async () => { - const spotlight = createPipeline({ + const spotlight = createTelemetryPipeline({ onDiagnostic: () => { throw new Error("observer failed"); }, @@ -293,7 +293,7 @@ describe("Node Telemetry adapter", () => { }); test("reset from a subscriber cannot repopulate raw payload history", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); let reset = false; spotlight.subscribe(() => { if (!reset && spotlight.getEnvelopes().length > 0) { @@ -309,7 +309,7 @@ describe("Node Telemetry adapter", () => { }); test("dispose clears raw payloads and prevents subsequent ingestion", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const result = await spotlight.ingestPayload({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); spotlight.dispose(); @@ -325,7 +325,7 @@ describe("bounded payload ingestion", () => { ["deflate", deflateSync], ["br", brotliCompressSync], ] as const)("rejects %s expansion at the decompressor output limit", async (encoding, compress) => { - const pipeline = createPipeline({ maxDecodedBytes: 128 }); + const pipeline = createTelemetryPipeline({ maxDecodedBytes: 128 }); const result = await pipeline.ingestPayload({ body: compress(Buffer.alloc(4096, 65)), encoding }); expect(result).toMatchObject({ status: "invalid", @@ -337,13 +337,13 @@ describe("bounded payload ingestion", () => { }); test("enforces input and uncompressed decoded byte limits", async () => { - const inputLimited = createPipeline({ maxPayloadBytes: 10 }); + const inputLimited = createTelemetryPipeline({ maxPayloadBytes: 10 }); expect(await inputLimited.ingestPayload({ body: fixture })).toMatchObject({ status: "invalid", diagnostics: [{ code: "payload_too_large" }], }); inputLimited.dispose(); - const decodedLimited = createPipeline({ maxDecodedBytes: 10 }); + const decodedLimited = createTelemetryPipeline({ maxDecodedBytes: 10 }); expect(await decodedLimited.ingestPayload({ body: fixture })).toMatchObject({ status: "invalid", diagnostics: [{ code: "decoded_payload_too_large" }], @@ -352,7 +352,7 @@ describe("bounded payload ingestion", () => { }); test("evicts raw bytes by total decoded size before notifying subscribers", async () => { - const pipeline = createPipeline({ maxRawBytes: fixture.length }); + const pipeline = createTelemetryPipeline({ decodedHistory: { maxBytes: fixture.length } }); const first = await pipeline.ingestPayload({ body: fixture }); if (first.status !== "processed") throw new Error("Expected processed envelope"); const observations: unknown[] = []; @@ -367,7 +367,7 @@ describe("bounded payload ingestion", () => { }); test("zero raw byte budget disables decoded history", async () => { - const pipeline = createPipeline({ maxRawBytes: 0 }); + const pipeline = createTelemetryPipeline({ decodedHistory: { maxBytes: 0 } }); const result = await pipeline.ingestPayload({ body: fixture }); if (result.status !== "processed") throw new Error("Expected processed envelope"); expect(pipeline.getDecodedPayload(result.envelopeId)).toBeUndefined(); @@ -375,7 +375,7 @@ describe("bounded payload ingestion", () => { }); test("forwards delivery identity and abort signals to processing", async () => { - const pipeline = createPipeline(); + const pipeline = createTelemetryPipeline(); const first = await pipeline.ingestPayload({ body: fixture, deliveryId: "wire-delivery" }); const duplicate = await pipeline.ingestPayload({ body: fixture, deliveryId: "wire-delivery" }); expect(first.status).toBe("processed"); @@ -391,7 +391,7 @@ describe("bounded payload ingestion", () => { }); test.each(["reset", "dispose"] as const)("%s cancels asynchronous decompression", async action => { - const pipeline = createPipeline(); + const pipeline = createTelemetryPipeline(); const pending = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); await Promise.resolve(); pipeline[action](); @@ -401,7 +401,7 @@ describe("bounded payload ingestion", () => { }); test("bounds concurrently admitted payloads and drain includes decoding", async () => { - const pipeline = createPipeline({ maxPending: 1 }); + const pipeline = createTelemetryPipeline({ maxPending: 1 }); const first = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); expect(await pipeline.ingestPayload({ body: fixture })).toMatchObject({ status: "ignored", reason: "queue_full" }); await pipeline.drain(); @@ -411,8 +411,8 @@ describe("bounded payload ingestion", () => { }); test("payload batches retain order and isolate invalid entries", async () => { - const pipeline = createPipeline(); - const results = await pipeline.ingestPayloadBatch([ + const pipeline = createTelemetryPipeline(); + const results = await pipeline.ingestPayloads([ { body: fixture, deliveryId: "first" }, { body: "invalid" }, { body: fixture, deliveryId: "first" }, @@ -425,7 +425,7 @@ describe("bounded payload ingestion", () => { describe("payload lifecycle boundaries", () => { test("delivery IDs key decoded history before public subscriptions", async () => { - const pipeline = createPipeline(); + const pipeline = createTelemetryPipeline(); const observed: unknown[] = []; pipeline.subscribe(snapshot => { const id = String(snapshot.envelopes[0][0].__spotlight_envelope_id); @@ -447,7 +447,7 @@ describe("payload lifecycle boundaries", () => { release = resolve; }); const controller = new AbortController(); - const pipeline = createPipeline({ + const pipeline = createTelemetryPipeline({ enrichStacktrace: async () => { start(); await gate; @@ -470,7 +470,7 @@ describe("payload lifecycle boundaries", () => { }); test("aborting decompression does not retain or process the decoded output", async () => { - const pipeline = createPipeline(); + const pipeline = createTelemetryPipeline(); const controller = new AbortController(); const pending = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip", signal: controller.signal }); await Promise.resolve(); @@ -480,21 +480,23 @@ describe("payload lifecycle boundaries", () => { pipeline.dispose(); }); - test.each(["maxPayloadBytes", "maxDecodedBytes", "maxRawBytes", "rawBufferSize"] as const)( + test.each(["maxPayloadBytes", "maxDecodedBytes", "maxBytes", "maxEntries"] as const)( "validates %s configuration", name => { for (const invalid of [-1, 0.1, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1]) { - expect(() => createPipeline({ [name]: invalid })).toThrow(RangeError); + const options = + name === "maxBytes" || name === "maxEntries" ? { decodedHistory: { [name]: invalid } } : { [name]: invalid }; + expect(() => createTelemetryPipeline(options)).toThrow(RangeError); } if (name === "maxPayloadBytes" || name === "maxDecodedBytes") { - expect(() => createPipeline({ [name]: 0 })).toThrow(RangeError); + expect(() => createTelemetryPipeline({ [name]: 0 })).toThrow(RangeError); } }, ); }); test("Node pending stats include queued decode work and reset releases admission capacity", async () => { - const pipeline = createPipeline({ maxPending: 1 }); + const pipeline = createTelemetryPipeline({ maxPending: 1 }); const old = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); expect(pipeline.getStats().pending).toBe(1); await Promise.resolve(); @@ -517,7 +519,7 @@ test("Node drain excludes payloads admitted after its barrier", async () => { const gate = new Promise(resolve => { release = resolve; }); - const pipeline = createPipeline({ + const pipeline = createTelemetryPipeline({ enrichStacktrace: async () => { start(); await gate; @@ -538,8 +540,8 @@ test("Node drain excludes payloads admitted after its barrier", async () => { }); test("drain includes every entry of an already-submitted payload batch", async () => { - const pipeline = createPipeline({ maxPending: 1 }); - const batch = pipeline.ingestPayloadBatch([ + const pipeline = createTelemetryPipeline({ maxPending: 1 }); + const batch = pipeline.ingestPayloads([ { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-second" }, ]); @@ -550,8 +552,8 @@ test("drain includes every entry of an already-submitted payload batch", async ( }); test("reset cancels the remaining entries of a submitted payload batch", async () => { - const pipeline = createPipeline(); - const batch = pipeline.ingestPayloadBatch([ + const pipeline = createTelemetryPipeline(); + const batch = pipeline.ingestPayloads([ { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, { body: fixture, deliveryId: "batch-second" }, ]); @@ -576,13 +578,13 @@ test("a queued payload batch stays ahead of later payload work", async () => { const gate = new Promise(resolve => { release = resolve; }); - const pipeline = createPipeline({ + const pipeline = createTelemetryPipeline({ enrichStacktrace: async () => { start(); await gate; }, }); - const batch = pipeline.ingestPayloadBatch([ + const batch = pipeline.ingestPayloads([ { body: gzipSync(fixture), encoding: "gzip", deliveryId: "batch-first" }, { body: fixture, deliveryId: "batch-second" }, ]); @@ -605,12 +607,12 @@ describe("one Node admission queue for every input method", () => { [[{ type: "event" }, { event_id: id, timestamp: 1, ...(error ? { exception: { values: [] } } : {}) }]], ]; - test.each(["ingestEnvelope", "ingestBatch"] as const)("wire admission bounds later %s calls", async method => { - const pipeline = createPipeline({ maxPending: 1 }); + test.each(["ingestEnvelope", "ingestEnvelopes"] as const)("wire admission bounds later %s calls", async method => { + const pipeline = createTelemetryPipeline({ maxPending: 1 }); const wire = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); const input = parsedEvent("later-parsed"); const later = - method === "ingestEnvelope" ? await pipeline.ingestEnvelope(input) : (await pipeline.ingestBatch([input]))[0]; + method === "ingestEnvelope" ? await pipeline.ingestEnvelope(input) : (await pipeline.ingestEnvelopes([input]))[0]; expect(later).toMatchObject({ status: "ignored", reason: "queue_full" }); expect(pipeline.getStats().pending).toBe(1); expect((await wire).status).toBe("processed"); @@ -618,24 +620,19 @@ describe("one Node admission queue for every input method", () => { pipeline.dispose(); }); - test.each(["ingestPayload", "ingestPayloadBatch"] as const)( - "parsed admission bounds later %s calls", - async method => { - const pipeline = createPipeline({ maxPending: 1 }); - const parsed = pipeline.ingestEnvelope(parsedEvent("first-parsed")); - const input = { body: fixture }; - const later = - method === "ingestPayload" - ? await pipeline.ingestPayload(input) - : (await pipeline.ingestPayloadBatch([input]))[0]; - expect(later).toMatchObject({ status: "ignored", reason: "queue_full" }); - expect(pipeline.getStats().pending).toBe(1); - expect((await parsed).status).toBe("processed"); - pipeline.dispose(); - }, - ); + test.each(["ingestPayload", "ingestPayloads"] as const)("parsed admission bounds later %s calls", async method => { + const pipeline = createTelemetryPipeline({ maxPending: 1 }); + const parsed = pipeline.ingestEnvelope(parsedEvent("first-parsed")); + const input = { body: fixture }; + const later = + method === "ingestPayload" ? await pipeline.ingestPayload(input) : (await pipeline.ingestPayloads([input]))[0]; + expect(later).toMatchObject({ status: "ignored", reason: "queue_full" }); + expect(pipeline.getStats().pending).toBe(1); + expect((await parsed).status).toBe("processed"); + pipeline.dispose(); + }); - test.each(["ingestEnvelope", "ingestBatch"] as const)("wire drain excludes a later gated %s", async method => { + test.each(["ingestEnvelope", "ingestEnvelopes"] as const)("wire drain excludes a later gated %s", async method => { let start!: () => void; let release!: () => void; const started = new Promise(resolve => { @@ -644,7 +641,7 @@ describe("one Node admission queue for every input method", () => { const gate = new Promise(resolve => { release = resolve; }); - const pipeline = createPipeline({ + const pipeline = createTelemetryPipeline({ enrichStacktrace: async () => { start(); await gate; @@ -653,10 +650,10 @@ describe("one Node admission queue for every input method", () => { const wire = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); const drained = pipeline.drain(); const input = parsedEvent("later-gated-parsed", true); - const later = method === "ingestEnvelope" ? pipeline.ingestEnvelope(input) : pipeline.ingestBatch([input]); + const later = method === "ingestEnvelope" ? pipeline.ingestEnvelope(input) : pipeline.ingestEnvelopes([input]); await started; // The wire job must commit first; its already-captured barrier must settle while later work is gated. - expect(pipeline.getQuerySummaries()).toHaveLength(1); + expect(pipeline.getDatabaseQuerySummaries()).toHaveLength(1); await drained; expect((await wire).status).toBe("processed"); release(); @@ -665,12 +662,12 @@ describe("one Node admission queue for every input method", () => { }); test("parsed batches retain order ahead of later wire jobs", async () => { - const pipeline = createPipeline(); + const pipeline = createTelemetryPipeline(); const seen: string[] = []; pipeline.subscribe((_snapshot, change) => { if (change.type === "ingest") seen.push(...change.envelopeIds); }); - const batch = pipeline.ingestBatch([ + const batch = pipeline.ingestEnvelopes([ [{ __spotlight_envelope_id: "parsed-first" }, []], [{ __spotlight_envelope_id: "parsed-second" }, []], ]); @@ -683,7 +680,7 @@ describe("one Node admission queue for every input method", () => { }); test("copied standalone parsed inputs retain weak object identity across calls and batches", async () => { - const pipeline = createPipeline(); + const pipeline = createTelemetryPipeline(); const input = parsedEvent("copied-input"); const pending = pipeline.ingestEnvelope(input); input[1] = []; @@ -696,14 +693,14 @@ describe("one Node admission queue for every input method", () => { reason: "duplicate", envelopeId: "envelopeId" in first ? first.envelopeId : "missing", }); - expect((await pipeline.ingestBatch([input, input])).map(item => item.status)).toEqual(["ignored", "ignored"]); + expect((await pipeline.ingestEnvelopes([input, input])).map(item => item.status)).toEqual(["ignored", "ignored"]); expect(pipeline.getEnvelopes()).toHaveLength(1); expect(input[0]).not.toHaveProperty("__spotlight_envelope_id"); pipeline.dispose(); }); test("prune calls coalesce without exceeding a full shared queue", async () => { - const pipeline = createPipeline({ maxPending: 1 }); + const pipeline = createTelemetryPipeline({ maxPending: 1 }); const wire = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); const first = pipeline.prune(); const second = pipeline.prune(); @@ -716,10 +713,10 @@ describe("one Node admission queue for every input method", () => { }); test("parsed batch clone failures produce per-item receipts and preserve later inputs", async () => { - const pipeline = createPipeline(); + const pipeline = createTelemetryPipeline(); const uncloneable = [{ bad: () => {} }, []]; const good = [{}, [[{ type: "event" }, { event_id: "after-clone-failure", timestamp: 1 }]]]; - const results = await pipeline.ingestBatch([uncloneable, good, good]); + const results = await pipeline.ingestEnvelopes([uncloneable, good, good]); expect(results).toMatchObject([ { status: "invalid", diagnostics: [{ code: "invalid_envelope" }] }, { status: "processed" }, @@ -731,10 +728,10 @@ test("parsed batch clone failures produce per-item receipts and preserve later i }); test("reset invalidates queued parsed jobs and the remaining parsed batch entries", async () => { - const pipeline = createPipeline(); + const pipeline = createTelemetryPipeline(); const wire = pipeline.ingestPayload({ body: gzipSync(fixture), encoding: "gzip" }); const parsed = pipeline.ingestEnvelope([{}, []]); - const batch = pipeline.ingestBatch([ + const batch = pipeline.ingestEnvelopes([ [{}, []], [{}, []], ]); @@ -753,7 +750,7 @@ test("reset invalidates queued parsed jobs and the remaining parsed batch entrie test("diagnostic callbacks cannot bypass shared admission during core validation", async () => { const later: Promise[] = []; const observed: number[] = []; - const pipeline = createPipeline({ + const pipeline = createTelemetryPipeline({ maxPending: 1, onDiagnostic() { observed.push(pipeline.getStats().pending); diff --git a/packages/spotlight/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts index 3b7108139..4a9e9e123 100644 --- a/packages/spotlight/src/sdk/node.ts +++ b/packages/spotlight/src/sdk/node.ts @@ -5,23 +5,26 @@ import type { BatchOptions, DiagnosticCode, IgnoreReason, IngestOptions } from " import { decodeEnvelope } from "../telemetry/envelope/codec.ts"; import { normalizeEnvelope, validateEnvelope } from "../telemetry/internal/envelopes.ts"; import { diagnostic, receipt, reportDiagnostic } from "../telemetry/internal/results.ts"; +import { validateRetention } from "../telemetry/internal/retention.ts"; import { generateUuidv4 } from "../telemetry/uuid.ts"; -import { type IngestResult, type Ingester, type IngesterOptions, createIngester } from "./core.ts"; +import { type IngestResult, type TelemetryStore, type TelemetryStoreOptions, createTelemetryStore } from "./core.ts"; -export type { IngesterSnapshot as PipelineSnapshot } from "./core.ts"; +export type { TelemetrySnapshot } from "./core.ts"; -export type PipelineOptions = IngesterOptions & { - /** Maximum retained decoded payloads (default 500); zero disables storage. */ - rawBufferSize?: number; +export type TelemetryPipelineOptions = TelemetryStoreOptions & { + decodedHistory?: { + /** Maximum retained decoded payloads (default 500); zero disables storage. */ + maxEntries?: number; + /** Total decoded payload history budget (default 32 MiB); zero disables storage. */ + maxBytes?: number; + }; /** Maximum input body size before decompression. Default: 8 MiB. */ maxPayloadBytes?: number; /** Maximum body size after decompression. Default: 32 MiB. */ maxDecodedBytes?: number; - /** Total decoded payload history budget (default 32 MiB); zero disables storage. */ - maxRawBytes?: number; }; -export type PipelineInput = { +export type TelemetryPayload = { body: Uint8Array | string; contentType?: string; encoding?: string; @@ -30,11 +33,11 @@ export type PipelineInput = { signal?: AbortSignal; }; -export type RawPayload = { body: Buffer; contentType: string }; -export type Pipeline = Ingester & { - ingestPayload(input: PipelineInput): Promise; - ingestPayloadBatch(inputs: readonly PipelineInput[]): Promise; - getDecodedPayload(envelopeId: string): RawPayload | undefined; +export type DecodedPayload = { body: Buffer; contentType: string }; +export type TelemetryPipeline = TelemetryStore & { + ingestPayload(input: TelemetryPayload): Promise; + ingestPayloads(inputs: readonly TelemetryPayload[]): Promise; + getDecodedPayload(envelopeId: string): DecodedPayload | undefined; }; const envelopeContentType = "application/x-sentry-envelope"; @@ -47,49 +50,49 @@ function limit(name: string, value: number, allowZero = false): number { return value; } -/** Decode bounded serialized Sentry envelopes, then process them through the shared ingester. */ -export function createPipeline(options: PipelineOptions = {}): Pipeline { - const rawBufferSize = limit("rawBufferSize", options.rawBufferSize ?? 500, true); - const maxRawBytes = limit("maxRawBytes", options.maxRawBytes ?? 32 * 1024 * 1024, true); +/** Decode bounded serialized Sentry envelopes, then process them through the shared telemetry store. */ +export function createTelemetryPipeline(options: TelemetryPipelineOptions = {}): TelemetryPipeline { + const maxHistoryEntries = limit("decodedHistory.maxEntries", options.decodedHistory?.maxEntries ?? 500, true); + const maxHistoryBytes = limit("decodedHistory.maxBytes", options.decodedHistory?.maxBytes ?? 32 * 1024 * 1024, true); const maxPayloadBytes = limit("maxPayloadBytes", options.maxPayloadBytes ?? 8 * 1024 * 1024); const maxDecodedBytes = limit("maxDecodedBytes", options.maxDecodedBytes ?? 32 * 1024 * 1024); const maxPending = limit("maxPending", options.maxPending ?? 64); const maxEnvelopeBytes = options.maxEnvelopeBytes ?? 32 * 1024 * 1024; - const telemetry = createIngester(options); - const rawPayloads = new Map(); - const stagedPayloads = new Map(); + const telemetry = createTelemetryStore(options); + const decodedPayloads = new Map(); + const stagedPayloads = new Map(); const pending = new Set>(); const decodeControllers = new Set(); const coreAdmissions = new Set(); const identities = new WeakMap(); - let rawBytes = 0; + let decodedBytes = 0; let generation = 0; let disposed = false; let queue = Promise.resolve(); let pruning: Promise | undefined; - function retainRawPayload(envelopeId: string, payload: RawPayload) { - // Oversized individual payloads are processed but cannot enter the raw history. - if (rawBufferSize === 0 || payload.body.length > maxRawBytes || maxRawBytes === 0) return; - const previous = rawPayloads.get(envelopeId); - if (previous) rawBytes -= previous.body.length; - rawPayloads.set(envelopeId, payload); - rawBytes += payload.body.length; - while (rawPayloads.size > rawBufferSize || rawBytes > maxRawBytes) { - const oldest = rawPayloads.keys().next().value!; - rawBytes -= rawPayloads.get(oldest)!.body.length; - rawPayloads.delete(oldest); + function retainDecodedPayload(envelopeId: string, payload: DecodedPayload) { + // Oversized individual payloads are processed but cannot enter the decoded history. + if (maxHistoryEntries === 0 || payload.body.length > maxHistoryBytes || maxHistoryBytes === 0) return; + const previous = decodedPayloads.get(envelopeId); + if (previous) decodedBytes -= previous.body.length; + decodedPayloads.set(envelopeId, payload); + decodedBytes += payload.body.length; + while (decodedPayloads.size > maxHistoryEntries || decodedBytes > maxHistoryBytes) { + const oldest = decodedPayloads.keys().next().value!; + decodedBytes -= decodedPayloads.get(oldest)!.body.length; + decodedPayloads.delete(oldest); } } - // Register before public listeners so raw history commits with the core snapshot. + // Register before public listeners so decoded history commits with the core snapshot. telemetry.subscribe(snapshot => { for (const envelope of snapshot.envelopes) { const envelopeId = String(envelope[0].__spotlight_envelope_id); const payload = stagedPayloads.get(envelopeId); if (!payload) continue; stagedPayloads.delete(envelopeId); - retainRawPayload(envelopeId, payload); + retainDecodedPayload(envelopeId, payload); } }); @@ -103,7 +106,7 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { } async function processPayload( - input: PipelineInput, + input: TelemetryPayload, body: Buffer, contentType: string, encoding: string | undefined, @@ -180,7 +183,8 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { const parsedEnvelope = envelope as Envelope; parsedEnvelope[0].__spotlight_envelope_id = envelopeId; const payload = { body, contentType }; - if (rawBufferSize > 0 && maxRawBytes > 0 && body.length <= maxRawBytes) stagedPayloads.set(envelopeId, payload); + if (maxHistoryEntries > 0 && maxHistoryBytes > 0 && body.length <= maxHistoryBytes) + stagedPayloads.set(envelopeId, payload); try { const result = await withCoreAdmission(() => telemetry.ingestEnvelope(parsedEnvelope, { @@ -198,7 +202,7 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { stagedPayloads.get(envelopeId) === payload ) { // Raw history is independent of processed retention, including immediate age/byte eviction. - retainRawPayload(result.envelopeId, payload); + retainDecodedPayload(result.envelopeId, payload); } return result; } finally { @@ -255,8 +259,8 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { return Math.max(pending.size, telemetry.getStats().pending + pending.size - coreAdmissions.size); } - type PreparedPayload = { input: PipelineInput; body: Buffer; contentType: string; encoding?: string }; - function preparePayload(input: PipelineInput): PreparedPayload | IngestResult { + type PreparedPayload = { input: TelemetryPayload; body: Buffer; contentType: string; encoding?: string }; + function preparePayload(input: TelemetryPayload): PreparedPayload | IngestResult { if (!input || (typeof input.body !== "string" && !(input.body instanceof Uint8Array))) { return invalid("invalid_body", "Envelope body must be a string or Uint8Array"); } @@ -297,7 +301,7 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { return operation; } - function ingestPayload(input: PipelineInput): Promise { + function ingestPayload(input: TelemetryPayload): Promise { if (disposed) return Promise.resolve(ignored("disposed")); if (input?.signal?.aborted) return Promise.resolve(ignored("aborted")); if (pendingCount() >= maxPending) return Promise.resolve(ignored("queue_full")); @@ -319,7 +323,10 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { return enqueue(epoch => processEnvelope(prepared, epoch)); } - function ingestBatch(inputs: Iterable, batchOptions: BatchOptions = {}): Promise { + function ingestEnvelopes( + inputs: Iterable, + batchOptions: BatchOptions = {}, + ): Promise { const signal = batchOptions.signal; const rejected = disposed ? "disposed" : pendingCount() >= maxPending ? "queue_full" : undefined; if (rejected) return Promise.resolve(Object.freeze(Array.from(inputs, () => ignored(rejected)))); @@ -340,9 +347,9 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { return { ...telemetry, ingestEnvelope, - ingestBatch, + ingestEnvelopes, ingestPayload, - ingestPayloadBatch(inputs) { + ingestPayloads(inputs) { const rejected = disposed ? "disposed" : pendingCount() >= maxPending ? "queue_full" : undefined; const entries = Array.from(inputs); if (rejected) return Promise.resolve(Object.freeze(entries.map(() => ignored(rejected)))); @@ -366,7 +373,7 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { }); }, getDecodedPayload(envelopeId) { - const payload = rawPayloads.get(envelopeId); + const payload = decodedPayloads.get(envelopeId); return payload && { body: Buffer.from(payload.body), contentType: payload.contentType }; }, getStats() { @@ -379,6 +386,17 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { const coreBarrier = telemetry.drain(); await Promise.all([wireBarrier, coreBarrier]); }, + setRetention(update) { + if (disposed) return Promise.resolve(); + const next = validateRetention(update); + const epoch = generation; + // Retention changes share payload ordering without consuming an admission slot. + const operation = queue.then(async () => { + if (!disposed && epoch === generation) await telemetry.setRetention(next); + }); + queue = operation.catch(() => {}); + return operation; + }, prune() { if (disposed) return Promise.resolve(); if (pruning) return pruning; @@ -402,9 +420,9 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { pending.clear(); coreAdmissions.clear(); for (const controller of decodeControllers) controller.abort(); - rawPayloads.clear(); + decodedPayloads.clear(); stagedPayloads.clear(); - rawBytes = 0; + decodedBytes = 0; telemetry.reset(); }, dispose() { @@ -415,9 +433,9 @@ export function createPipeline(options: PipelineOptions = {}): Pipeline { pending.clear(); coreAdmissions.clear(); for (const controller of decodeControllers) controller.abort(); - rawPayloads.clear(); + decodedPayloads.clear(); stagedPayloads.clear(); - rawBytes = 0; + decodedBytes = 0; telemetry.dispose(); }, }; diff --git a/packages/spotlight/src/server/sdk.spec.ts b/packages/spotlight/src/server/sdk.spec.ts index cf45b4652..d54834ff4 100644 --- a/packages/spotlight/src/server/sdk.spec.ts +++ b/packages/spotlight/src/server/sdk.spec.ts @@ -3,7 +3,7 @@ import { gzipSync } from "node:zlib"; // @vitest-environment node import type { Envelope } from "@sentry/core"; import { describe, expect, test } from "vitest"; -import { createPipeline } from "../sdk/node.ts"; +import { createTelemetryPipeline } from "../sdk/node.ts"; import * as sdk from "./sdk.ts"; const traceId = "a9eab6940aba4ae7bf2f8ac5a54c6a2f"; @@ -34,10 +34,10 @@ function span(overrides: Record = {}) { describe("shared Telemetry SDK", () => { test("ingests compressed v2 spans into traces and query summaries without a browser", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const result = await spotlight.ingestPayload({ body: gzipSync(fixture), encoding: "gzip", contentType }); if (result.status !== "processed") throw new Error("Expected processed envelope"); - expect(spotlight.getQuerySummaries()).toMatchObject([ + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([ { count: 1, totalDurationMs: 125, @@ -77,18 +77,18 @@ describe("shared Telemetry SDK", () => { ], }); const before = JSON.stringify(input); - const first = createPipeline(); - const second = createPipeline(); + const first = createTelemetryPipeline(); + const second = createTelemetryPipeline(); await first.ingestEnvelope(input); await second.ingestEnvelope(input); await first.ingestEnvelope(input); expect(JSON.stringify(input)).toBe(before); - expect(first.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); - expect(second.getQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(first.getDatabaseQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(second.getDatabaseQuerySummaries()).toMatchObject([{ count: 1, totalDurationMs: 125 }]); }); test("keeps child spans when parents arrive later and deduplicates repeated delivery", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); await spotlight.ingestEnvelope(envelope("span", { version: 2, items: [span()] })); await spotlight.ingestEnvelope( envelope( @@ -103,7 +103,7 @@ describe("shared Telemetry SDK", () => { ), ); await spotlight.ingestEnvelope(envelope("span", { version: 2, items: [span()] }, "redelivery")); - expect(spotlight.getQuerySummaries()).toMatchObject([ + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([ { count: 1, description: "SELECT 1", averageDurationMs: 125 }, ]); expect(spotlight.getTrace(traceId)?.spanTree[0].children?.map(s => s.span_id)).toEqual([queryId]); @@ -111,19 +111,19 @@ describe("shared Telemetry SDK", () => { }); test("can scope queries to a trace without merging spans that share an ID", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); await spotlight.ingestEnvelope( envelope("span", { version: 2, items: [span(), span({ trace_id: "other-trace" })] }), ); - expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 2, totalDurationMs: 250 }]); - expect(spotlight.getQuerySummaries({ traceId })).toMatchObject([{ count: 1, totalDurationMs: 125 }]); + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([{ count: 2, totalDurationMs: 250 }]); + expect(spotlight.getDatabaseQuerySummaries({ traceId })).toMatchObject([{ count: 1, totalDurationMs: 125 }]); expect(spotlight.getSpans({ traceId: "absent" })).toEqual([]); }); test.each([true, false])( "merges a legacy transaction and streamed segment for the same span (legacy first: %s)", async legacyFirst => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const legacy = envelope( "transaction", { @@ -171,25 +171,25 @@ describe("shared Telemetry SDK", () => { expect(trace.rootTransactionName).toBe("request"); expect(trace.spans.get(rootId)?.op).toBe("http.server"); expect(trace.spans.get(rootId)?.data).toMatchObject({ legacy: true, streamed: "yes" }); - expect(spotlight.getQuerySummaries()).toMatchObject([{ count: 2, totalDurationMs: 150 }]); + expect(spotlight.getDatabaseQuerySummaries()).toMatchObject([{ count: 2, totalDurationMs: 150 }]); }, ); test("isolates instance state and resets buffer and processed data together", async () => { - const first = createPipeline(); - const second = createPipeline(); + const first = createTelemetryPipeline(); + const second = createTelemetryPipeline(); const ingested = await first.ingestPayload({ body: fixture, contentType }); expect(second.getTraces()).toEqual([]); expect(second.getSdks()).toEqual([]); first.reset(); expect(first.getTraces()).toEqual([]); - expect(first.getQuerySummaries()).toEqual([]); + expect(first.getDatabaseQuerySummaries()).toEqual([]); expect(first.getEnvelopes()).toEqual([]); if (ingested.status === "processed") expect(first.getDecodedPayload(ingested.envelopeId)).toBeUndefined(); }); test("awaits optional stacktrace enrichment before exposing errors and trace correlation", async () => { - const spotlight = createPipeline({ + const spotlight = createTelemetryPipeline({ enrichStacktrace: async event => { await Promise.resolve(); event.exception!.values![0].stacktrace!.frames[0].context_line = "throw new Error()"; @@ -216,7 +216,7 @@ describe("shared Telemetry SDK", () => { const gate = new Promise(resolve => { release = resolve; }); - const spotlight = createPipeline({ enrichStacktrace: () => gate }); + const spotlight = createTelemetryPipeline({ enrichStacktrace: () => gate }); const pending = spotlight.ingestEnvelope( envelope("event", { event_id: "error", @@ -236,7 +236,7 @@ describe("shared Telemetry SDK", () => { test.each(["sdks", "logs", "events"] as const)( "reset from a %s subscriber cancels the current ingestion", async key => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); let reset = false; spotlight.subscribe(state => { if (!reset && state[key].length > 0) { @@ -264,7 +264,7 @@ describe("shared Telemetry SDK", () => { ); test("normalizes logs and correlates them to traces", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); await spotlight.ingestEnvelope( envelope("log", { type: "log", @@ -275,7 +275,7 @@ describe("shared Telemetry SDK", () => { }); test("redelivery of the same idless parsed envelope does not duplicate logs", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const input = [ {}, [[{ type: "log" }, { items: [{ timestamp: 1, body: "hello", level: "info", attributes: {} }] }]], @@ -292,9 +292,9 @@ describe("shared Telemetry SDK", () => { }); test("notifies subscribers with completed state and supports unsubscribing", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const observed: number[] = []; - const unsubscribe = spotlight.subscribe(() => observed.push(spotlight.getQuerySummaries().length)); + const unsubscribe = spotlight.subscribe(() => observed.push(spotlight.getDatabaseQuerySummaries().length)); await spotlight.ingestPayload({ body: fixture, contentType }); expect(observed.at(-1)).toBe(1); unsubscribe(); @@ -314,7 +314,7 @@ describe("shared Telemetry SDK", () => { const gate = new Promise(resolve => { release = resolve; }); - const spotlight = createPipeline({ enrichStacktrace: () => gate }); + const spotlight = createTelemetryPipeline({ enrichStacktrace: () => gate }); const input = envelope("event", { event_id: "pending", type: "event", timestamp: 1, exception: { values: [] } }); const first = spotlight.ingestEnvelope(input); let secondFinished = false; @@ -329,7 +329,7 @@ describe("shared Telemetry SDK", () => { }); test("an unavailable enrichment adapter does not discard a captured error", async () => { - const spotlight = createPipeline({ + const spotlight = createTelemetryPipeline({ enrichStacktrace: async () => { throw new Error("offline"); }, @@ -341,7 +341,7 @@ describe("shared Telemetry SDK", () => { }); test("shares resource summaries, AI extraction, and web-vitals scoring", async () => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); await spotlight.ingestEnvelope( envelope("span", { version: 2, @@ -395,7 +395,7 @@ describe("shared Telemetry SDK", () => { test.each([true, false])( "associates continuous profiles with their trace (profile first: %s)", async profileFirst => { - const spotlight = createPipeline(); + const spotlight = createTelemetryPipeline(); const transaction = readFileSync( new URL("../../_fixtures/continuous_profiling/transaction.txt", import.meta.url), ); diff --git a/packages/spotlight/src/telemetry/attachments.spec.ts b/packages/spotlight/src/telemetry/attachments.spec.ts index 16d2b8ba4..4938a885e 100644 --- a/packages/spotlight/src/telemetry/attachments.spec.ts +++ b/packages/spotlight/src/telemetry/attachments.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { createIngester } from "../sdk/core.ts"; +import { createTelemetryStore } from "../sdk/core.ts"; describe("parsed protocol attachments", () => { test.each([ @@ -11,7 +11,7 @@ describe("parsed protocol attachments", () => { ] as const)( "associates %s with its event and retains canonical immutable data", async (_, payload, contentType, data) => { - const telemetry = createIngester(); + const telemetry = createTelemetryStore(); const input = [ {}, [ @@ -31,7 +31,7 @@ describe("parsed protocol attachments", () => { ); test("retains opaque unsupported items without discarding their accompanying event", async () => { - const telemetry = createIngester(); + const telemetry = createTelemetryStore(); const result = await telemetry.ingestEnvelope([ {}, [ @@ -52,7 +52,7 @@ describe("parsed protocol attachments", () => { }); test("continues to reject primitive payloads for supported telemetry items", async () => { - const telemetry = createIngester(); + const telemetry = createTelemetryStore(); const result = await telemetry.ingestEnvelope([{}, [[{ type: "event" }, "not a parsed event"]]]); expect(result.status).toBe("invalid"); expect(telemetry.getEnvelopes()).toEqual([]); diff --git a/packages/spotlight/src/telemetry/client.spec.ts b/packages/spotlight/src/telemetry/client.spec.ts index ba37442e5..eb4fa3277 100644 --- a/packages/spotlight/src/telemetry/client.spec.ts +++ b/packages/spotlight/src/telemetry/client.spec.ts @@ -26,26 +26,26 @@ function spans(id: string, traceId = "trace", count = 1) { afterEach(() => vi.useRealTimers()); test("canonical reads have consistent filters and explicit summary units", async () => { - const client = core.createIngester(); + const client = core.createTelemetryStore(); const result = await client.ingestEnvelope(spans("one")); expect(result).toMatchObject({ status: "processed", envelopeId: "one", diagnostics: [] }); - expect(client.getQuerySummaries({ traceId: "trace" })).toMatchObject([ + expect(client.getDatabaseQuerySummaries({ traceId: "trace" })).toMatchObject([ { count: 1, totalDurationMs: 125, averageDurationMs: 125 }, ]); expect(client.getTrace("trace")?.spans.size).toBe(1); expect(client.getSpans({ traceId: "absent" })).toEqual([]); - expect(client.getQuerySummaries({ traceId: "absent" })).toEqual([]); + expect(client.getDatabaseQuerySummaries({ traceId: "absent" })).toEqual([]); expect(client).not.toHaveProperty("store"); client.dispose(); }); test("subscribers see exactly one coherent update per ingestion", async () => { - const client = core.createIngester(); + const client = core.createTelemetryStore(); const seen: number[] = []; client.subscribe(snapshot => { expect(snapshot.events).toHaveLength(3); expect(snapshot.traces).toHaveLength(1); - expect(client.getQuerySummaries()[0].count).toBe(3); + expect(client.getDatabaseQuerySummaries()[0].count).toBe(3); seen.push(snapshot.revision); }); const initial = client.getSnapshot(); @@ -57,7 +57,7 @@ test("subscribers see exactly one coherent update per ingestion", async () => { }); test("returned events, nested maps and earlier snapshots cannot mutate engine state", async () => { - const client = core.createIngester(); + const client = core.createTelemetryStore(); await client.ingestEnvelope(spans("one")); const snapshot = client.getSnapshot(); const trace = client.getTrace("trace")!; @@ -70,12 +70,12 @@ test("returned events, nested maps and earlier snapshots cannot mutate engine st await client.ingestEnvelope(spans("two")); expect(snapshot.traces[0].spans.size).toBe(1); expect(client.getTrace("trace")?.spans.size).toBe(2); - expect(client.getQuerySummaries()[0].description).toBe("SELECT 1"); + expect(client.getDatabaseQuerySummaries()[0].description).toBe("SELECT 1"); client.dispose(); }); test("snapshots preserve own prototype-named JSON fields without inheriting their values", async () => { - const client = core.createIngester(); + const client = core.createTelemetryStore(); const input = JSON.parse( '[{"__spotlight_envelope_id":"proto"},[[{"type":"event"},{"event_id":"proto","timestamp":1,"extra":{"__proto__":{"unexpected":true},"kept":1}}]]]', ); @@ -91,7 +91,7 @@ test("snapshots preserve own prototype-named JSON fields without inheriting thei }); test("invalid and unsupported envelopes return diagnostics without changing the snapshot", async () => { - const client = core.createIngester(); + const client = core.createTelemetryStore(); const initial = client.getSnapshot(); expect(await client.ingestEnvelope(null)).toMatchObject({ status: "invalid", @@ -111,7 +111,7 @@ test("invalid and unsupported envelopes return diagnostics without changing the test("invalid parsed input reports the same diagnostic to the observer and caller", async () => { const diagnostics: string[] = []; - const client = core.createIngester({ onDiagnostic: diagnostic => diagnostics.push(diagnostic.code) }); + const client = core.createTelemetryStore({ onDiagnostic: diagnostic => diagnostics.push(diagnostic.code) }); const result = await client.ingestEnvelope(null); expect(diagnostics).toEqual(["invalid_envelope"]); expect(result.diagnostics.map(diagnostic => diagnostic.code)).toEqual(diagnostics); @@ -128,7 +128,7 @@ test("a rejected envelope waits for sibling enrichment and cannot poison later e entered = resolve; }); let enrichments = 0; - const client = core.createIngester({ + const client = core.createTelemetryStore({ enrichStacktrace: async event => { enrichments++; if (enrichments === 1) { @@ -163,7 +163,7 @@ test("a rejected envelope waits for sibling enrichment and cannot poison later e }); test("malformed spans have diagnostics while mixed valid spans still process", async () => { - const client = core.createIngester(); + const client = core.createTelemetryStore(); expect(await client.ingestEnvelope([{}, [[{ type: "span" }, { version: 2, items: [{}] }]]])).toMatchObject({ status: "invalid", diagnostics: [{ code: "invalid_span" }], @@ -180,8 +180,8 @@ test("malformed spans have diagnostics while mixed valid spans still process", a test("retention preserves identities of profile spans derived from retained envelopes", async () => { const { readFileSync } = await import("node:fs"); - const { createPipeline } = await import("../sdk/node.ts"); - const client = createPipeline({ retention: { maxEnvelopes: 3 } }); + const { createTelemetryPipeline } = await import("../sdk/node.ts"); + const client = createTelemetryPipeline({ retention: { maxEnvelopes: 3 } }); await client.ingestEnvelope(spans("evict-me")); for (const file of ["transaction.txt", "profile_chunk_1.txt"]) { const body = readFileSync(new URL(`../../_fixtures/continuous_profiling/${file}`, import.meta.url)); @@ -197,7 +197,7 @@ test("retention preserves identities of profile spans derived from retained enve }); test("count retention evicts related events, traces, logs and profiles coherently", async () => { - const client = core.createIngester({ retention: { maxEnvelopes: 1 } }); + const client = core.createTelemetryStore({ retention: { maxEnvelopes: 1 } }); await client.ingestEnvelope([ { __spotlight_envelope_id: "old" }, [ @@ -225,13 +225,13 @@ test("count retention evicts related events, traces, logs and profiles coherentl expect(client.getTrace("old")).toBeUndefined(); expect(client.getLogs({ traceId: "old" })).toEqual([]); expect(client.getEvents()).toHaveLength(1); - expect(client.getQuerySummaries()).toMatchObject([{ description: "SELECT 1", count: 1 }]); + expect(client.getDatabaseQuerySummaries()).toMatchObject([{ description: "SELECT 1", count: 1 }]); client.dispose(); }); test("retention replay preserves generated log IDs and does not repeat enrichment", async () => { let enrichments = 0; - const client = core.createIngester({ + const client = core.createTelemetryStore({ retention: { maxEnvelopes: 2 }, enrichStacktrace: async () => { enrichments++; @@ -266,7 +266,7 @@ test("duplicate error IDs have one deterministic enrichment owner across replay" entered = resolve; }); let calls = 0; - const client = core.createIngester({ + const client = core.createTelemetryStore({ retention: { maxEnvelopes: 2 }, enrichStacktrace: async () => { calls++; @@ -297,7 +297,7 @@ test("duplicate error IDs have one deterministic enrichment owner across replay" test("evicting an error owner enriches the retained duplicate from its own exception", async () => { const seen: string[] = []; - const client = core.createIngester({ + const client = core.createTelemetryStore({ retention: { maxEnvelopes: 3 }, enrichStacktrace: async event => { const exception = event.exception.values![0]; @@ -323,7 +323,7 @@ test("evicting an error owner enriches the retained duplicate from its own excep test("SDK last-seen times do not advance when retained envelopes replay", async () => { vi.useFakeTimers(); - const client = core.createIngester({ retention: { maxEnvelopes: 2 } }); + const client = core.createTelemetryStore({ retention: { maxEnvelopes: 2 } }); for (const [index, name] of ["old", "keep", "new"].entries()) { vi.setSystemTime(1000 * (index + 1)); await client.ingestEnvelope([ @@ -337,7 +337,7 @@ test("SDK last-seen times do not advance when retained envelopes replay", async test("age retention expires while idle and disposes its timer", async () => { vi.useFakeTimers(); - const client = core.createIngester({ retention: { maxAgeMs: 1000 } }); + const client = core.createTelemetryStore({ retention: { maxAgeMs: 1000 } }); await client.ingestEnvelope(spans("old")); await vi.advanceTimersByTimeAsync(1000); expect(client.getTraces()).toEqual([]); @@ -358,7 +358,7 @@ test("reset invalidates active and queued work while allowing immediate new inge const ready = new Promise(resolve => { entered = resolve; }); - const client = core.createIngester({ + const client = core.createTelemetryStore({ enrichStacktrace: () => { entered(); return gate; @@ -382,7 +382,7 @@ test("reset invalidates active and queued work while allowing immediate new inge }); test("subscribers can reset without a later notification restoring stale state", async () => { - const client = core.createIngester(); + const client = core.createTelemetryStore(); let reset = false; const observed: number[] = []; client.subscribe(snapshot => { diff --git a/packages/spotlight/src/telemetry/index.ts b/packages/spotlight/src/telemetry/index.ts index b24e2efe0..b4b86dca2 100644 --- a/packages/spotlight/src/telemetry/index.ts +++ b/packages/spotlight/src/telemetry/index.ts @@ -13,7 +13,8 @@ import { createEngine } from "./internal/engine.ts"; import { normalizeEnvelope, supportsEnvelope, validateEnvelope } from "./internal/envelopes.ts"; import { readonlyCopy } from "./internal/readonly.ts"; import { diagnostic as makeDiagnostic, receipt, reportDiagnostic } from "./internal/results.ts"; -import type { Ingester, IngesterOptions, IngesterSnapshot } from "./public.ts"; +import { validateRetention } from "./internal/retention.ts"; +import type { TelemetrySnapshot, TelemetryStore, TelemetryStoreOptions } from "./public.ts"; import { createTelemetryReader } from "./reader.ts"; import { normalizeSpanItem } from "./store/utils/spanProcessor.ts"; import type { SentryErrorEvent } from "./types.ts"; @@ -27,15 +28,14 @@ function positive(value: number, name: string) { } /** Portable processing with bounded work, immutable committed reads and explicit receipts. */ -export function createIngester(options: IngesterOptions = {}): Ingester { - const maxEnvelopes = positive(options.retention?.maxEnvelopes ?? 500, "retention.maxEnvelopes"); - const maxBytes = positive(options.retention?.maxBytes ?? 64 * MiB, "retention.maxBytes"); +export function createTelemetryStore(options: TelemetryStoreOptions = {}): TelemetryStore { + const retention = validateRetention(options.retention ?? {}); + let maxEnvelopes = retention.maxEnvelopes ?? 500; + let maxBytes = retention.maxBytes ?? 64 * MiB; const maxPending = positive(options.maxPending ?? 64, "maxPending"); const maxEnvelopeBytes = positive(options.maxEnvelopeBytes ?? 32 * MiB, "maxEnvelopeBytes"); const enrichmentTimeoutMs = positive(options.enrichmentTimeoutMs ?? 5000, "enrichmentTimeoutMs"); - const maxAgeMs = options.retention?.maxAgeMs; - if (maxAgeMs !== undefined && (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0)) - throw new RangeError("retention.maxAgeMs must be a positive finite number"); + let maxAgeMs = retention.maxAgeMs; let generation = 0; let disposed = false; let revision = 0; @@ -46,7 +46,7 @@ export function createIngester(options: IngesterOptions = {}): Ingester { let timer: ReturnType | undefined; let lifetime = new AbortController(); const identities = new WeakMap(); - const listeners = new Set<(snapshot: IngesterSnapshot, change: ChangeMetadata) => void>(); + const listeners = new Set<(snapshot: TelemetrySnapshot, change: ChangeMetadata) => void>(); let enrichmentCache = new Map(); let currentDiagnostics: Diagnostic[] = []; const stableCopies = new WeakMap(); @@ -176,7 +176,7 @@ export function createIngester(options: IngesterOptions = {}): Ingester { if (!stableCopies.has(value)) stableCopies.set(value, readonlyCopy(value)); return stableCopies.get(value) as T; } - function stateSnapshot(): IngesterSnapshot { + function stateSnapshot(): TelemetrySnapshot { const state = engine.store.getState(); return Object.freeze({ revision, @@ -425,10 +425,22 @@ export function createIngester(options: IngesterOptions = {}): Ingester { }; return withinEpoch === undefined ? enqueue(process) : process(withinEpoch); } - const client: Ingester = { + function queueRetentionPrune(): Promise { + const epoch = generation; + const operation = queue.then(async () => { + if (!active(epoch)) return; + currentDiagnostics = []; + const evicted = await enforceRetention(epoch); + if (active(epoch) && evicted.length) publish("evict", evicted); + if (active(epoch)) scheduleExpiry(); + }); + queue = operation.catch(() => {}); + return operation; + } + const client: TelemetryStore = { ...reader, ingestEnvelope: (input, inputOptions) => ingestOne(input, inputOptions), - ingestBatch(inputs, batchOptions = {}) { + ingestEnvelopes(inputs, batchOptions = {}) { const signal = batchOptions.signal; if (disposed || pending >= maxPending) { return Promise.resolve(Object.freeze(Array.from(inputs, () => ignored(disposed ? "disposed" : "queue_full")))); @@ -474,20 +486,21 @@ export function createIngester(options: IngesterOptions = {}): Ingester { pending = 0; publish("reset"); }, + setRetention(update) { + if (disposed) return Promise.resolve(); + const next = validateRetention(update); + if (next.maxEnvelopes !== undefined) maxEnvelopes = next.maxEnvelopes; + if (next.maxBytes !== undefined) maxBytes = next.maxBytes; + if (next.maxAgeMs !== undefined) maxAgeMs = next.maxAgeMs; + // A distinct barrier must cover work submitted after an earlier coalesced prune. + return queueRetentionPrune(); + }, prune() { if (disposed) return Promise.resolve(); if (pruning) return pruning; // Control work has one coalesced continuation outside ingestion admission. // Link it into the same barrier so even invalid/empty work cannot skip expiry. - const epoch = generation; - const operation = queue.then(async () => { - if (!active(epoch)) return; - currentDiagnostics = []; - const evicted = await enforceRetention(epoch); - if (active(epoch) && evicted.length) publish("evict", evicted); - if (active(epoch)) scheduleExpiry(); - }); - queue = operation.catch(() => {}); + const operation = queueRetentionPrune(); pruning = operation; void operation.then( () => { @@ -518,5 +531,5 @@ export function createIngester(options: IngesterOptions = {}): Ingester { }; return client; } -export type { Ingester, IngesterOptions, IngesterSnapshot } from "./public.ts"; +export type { TelemetryStore, TelemetryStoreOptions, TelemetrySnapshot } from "./public.ts"; export type { IngestResult, Diagnostic } from "./contracts.ts"; diff --git a/packages/spotlight/src/telemetry/internal/readonly.spec.ts b/packages/spotlight/src/telemetry/internal/readonly.spec.ts new file mode 100644 index 000000000..4997bdb09 --- /dev/null +++ b/packages/spotlight/src/telemetry/internal/readonly.spec.ts @@ -0,0 +1,26 @@ +import { expect, test } from "vitest"; +import { readonlyCopy } from "./readonly.ts"; + +test("copies deep graphs without recursion while preserving aliases and cycles", () => { + type Link = { next?: Link; root?: Link }; + const root: Link = {}; + let tail = root; + for (let i = 0; i < 10000; i++) { + tail.next = {}; + tail = tail.next; + } + tail.root = root; + const input = { root, aliases: new Map([[root, new Set([tail])]]) }; + const copied = readonlyCopy(input); + let current = copied.root; + for (let i = 0; i < 10000; i++) { + expect(Object.isFrozen(current)).toBe(true); + current = current.next!; + } + expect(current.root === copied.root).toBe(true); + expect(copied.aliases.get(copied.root)?.has(current)).toBe(true); + expect(() => (copied.aliases as Map>).clear()).toThrow(TypeError); + expect(() => (copied.aliases.get(copied.root) as Set).add(root)).toThrow(TypeError); + expect(root === copied.root).toBe(false); + expect(Object.isFrozen(root)).toBe(false); +}); diff --git a/packages/spotlight/src/telemetry/internal/readonly.ts b/packages/spotlight/src/telemetry/internal/readonly.ts index 5128718d3..884dec101 100644 --- a/packages/spotlight/src/telemetry/internal/readonly.ts +++ b/packages/spotlight/src/telemetry/internal/readonly.ts @@ -3,6 +3,7 @@ import type { DeepReadonly } from "../public.ts"; /** Copies the graph before freezing it; Map/Set mutation methods never expose the backing collection. */ export function readonlyCopy(input: T): DeepReadonly { const copies = new WeakMap(); + const pending: (() => void)[] = []; function copy(value: unknown): unknown { if (!value || typeof value !== "object") return value; if (copies.has(value)) return copies.get(value); @@ -39,21 +40,29 @@ export function readonlyCopy(input: T): DeepReadonly { }, }); copies.set(value, proxy); - if (value instanceof Map && target instanceof Map) - for (const [key, item] of value) target.set(copy(key), copy(item)); - else if (value instanceof Set && target instanceof Set) for (const item of value) target.add(copy(item)); + pending.push(() => { + if (value instanceof Map && target instanceof Map) + for (const [key, item] of value) target.set(copy(key), copy(item)); + else if (value instanceof Set && target instanceof Set) for (const item of value) target.add(copy(item)); + }); return proxy; } const result: Record | unknown[] = Array.isArray(value) ? [] : {}; copies.set(value, result); - for (const key of Object.keys(value)) - Object.defineProperty(result, key, { - value: copy((value as Record)[key]), - enumerable: true, - configurable: true, - writable: true, - }); - return Object.freeze(result); + pending.push(() => { + for (const key of Object.keys(value)) + Object.defineProperty(result, key, { + value: copy((value as Record)[key]), + enumerable: true, + configurable: true, + writable: true, + }); + Object.freeze(result); + }); + return result; } - return copy(input) as DeepReadonly; + const result = copy(input); + // Allocate each node once, then fill it from a worklist instead of the call stack. + for (let next = pending.pop(); next; next = pending.pop()) next(); + return result as DeepReadonly; } diff --git a/packages/spotlight/src/telemetry/internal/retention.ts b/packages/spotlight/src/telemetry/internal/retention.ts new file mode 100644 index 000000000..7130c75cd --- /dev/null +++ b/packages/spotlight/src/telemetry/internal/retention.ts @@ -0,0 +1,16 @@ +import type { RetentionOptions } from "../public.ts"; + +/** Copy and validate the complete partial update before any live limits change. */ +export function validateRetention(options: RetentionOptions): RetentionOptions { + const { maxEnvelopes, maxBytes, maxAgeMs } = options; + for (const [name, value] of [ + ["maxEnvelopes", maxEnvelopes], + ["maxBytes", maxBytes], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) + throw new RangeError(`retention.${name} must be a positive safe integer`); + } + if (maxAgeMs !== undefined && (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0)) + throw new RangeError("retention.maxAgeMs must be a positive finite number"); + return { maxEnvelopes, maxBytes, maxAgeMs }; +} diff --git a/packages/spotlight/src/telemetry/lifecycle.spec.ts b/packages/spotlight/src/telemetry/lifecycle.spec.ts index 06915c4d9..42bb16787 100644 --- a/packages/spotlight/src/telemetry/lifecycle.spec.ts +++ b/packages/spotlight/src/telemetry/lifecycle.spec.ts @@ -1,7 +1,7 @@ // @vitest-environment node import { afterEach, expect, test, vi } from "vitest"; -import { createPipeline } from "../sdk/node.ts"; -import { createIngester } from "./index.ts"; +import { createTelemetryPipeline } from "../sdk/node.ts"; +import { createTelemetryStore } from "./index.ts"; const event = (id = "event") => [{}, [[{ type: "event" }, { event_id: id, timestamp: 1 }]]]; const error = (id = "error") => [ @@ -14,11 +14,11 @@ test.each(["core", "node"] as const)("%s pruning after full-queue no-op work sti vi.useFakeTimers(); const client = runtime === "core" - ? createIngester({ maxPending: 1, retention: { maxAgeMs: 1000 } }) - : createPipeline({ maxPending: 1, retention: { maxAgeMs: 1000 } }); + ? createTelemetryStore({ maxPending: 1, retention: { maxAgeMs: 1000 } }) + : createTelemetryPipeline({ maxPending: 1, retention: { maxAgeMs: 1000 } }); await client.ingestEnvelope(event()); vi.setSystemTime(Date.now() + 10000); - const work = "ingestPayload" in client ? client.ingestPayload({ body: "bad" }) : client.ingestBatch([]); + const work = "ingestPayload" in client ? client.ingestPayload({ body: "bad" }) : client.ingestEnvelopes([]); await Promise.all([work, client.prune()]); expect(client.getEnvelopes()).toEqual([]); expect(client.getStats().pending).toBe(0); @@ -26,7 +26,7 @@ test.each(["core", "node"] as const)("%s pruning after full-queue no-op work sti }); test("pruning is coalesced and cannot bypass the configured pending limit", async () => { - const ingester = createIngester({ maxPending: 1 }); + const ingester = createTelemetryStore({ maxPending: 1 }); const operations = Array.from({ length: 100 }, () => ingester.prune()); expect(ingester.getStats().pending).toBeLessThanOrEqual(1); expect(operations[0]).toBe(operations[99]); @@ -37,7 +37,7 @@ test("pruning is coalesced and cannot bypass the configured pending limit", asyn test("large enrichment deadlines do not overflow Node timers", async () => { vi.useFakeTimers(); - const ingester = createIngester({ + const ingester = createTelemetryStore({ enrichmentTimeoutMs: 2147483648, enrichStacktrace: async () => { await new Promise(resolve => setTimeout(resolve, 10)); @@ -50,7 +50,7 @@ test("large enrichment deadlines do not overflow Node timers", async () => { }); test("receipts distinguish retained unsupported data and no-op duplicates", async () => { - const ingester = createIngester(); + const ingester = createTelemetryStore(); const first = await ingester.ingestEnvelope(event(), { deliveryId: "delivery" }); expect(first).toMatchObject({ status: "processed", @@ -77,8 +77,8 @@ test("receipts distinguish retained unsupported data and no-op duplicates", asyn }); test("batch ingestion preserves order and a drain barrier covers earlier work", async () => { - const ingester = createIngester(); - const batch = ingester.ingestBatch([event("one"), null, event("two")]); + const ingester = createTelemetryStore(); + const batch = ingester.ingestEnvelopes([event("one"), null, event("two")]); await ingester.drain(); expect((await batch).map(result => result.status)).toEqual(["processed", "invalid", "processed"]); expect(ingester.getEvents().map(item => item.event_id)).toEqual(["one", "two"]); @@ -93,7 +93,7 @@ test("batches are one queued operation and drain does not wait for later submiss const ready = new Promise(resolve => { enteredFirst = resolve; }); - const ingester = createIngester({ + const ingester = createTelemetryStore({ enrichStacktrace: async item => { if (item.event_id === "first") { enteredFirst(); @@ -107,7 +107,7 @@ test("batches are one queued operation and drain does not wait for later submiss }); }, }); - const batch = ingester.ingestBatch([error("first"), event("second")]); + const batch = ingester.ingestEnvelopes([error("first"), event("second")]); const barrier = ingester.drain(); const later = ingester.ingestEnvelope(error("later")); await ready; @@ -125,7 +125,7 @@ test("batches are one queued operation and drain does not wait for later submiss test("a timed out hook cannot mutate committed telemetry or block later work", async () => { vi.useFakeTimers(); let finish!: () => void; - const ingester = createIngester({ + const ingester = createTelemetryStore({ enrichmentTimeoutMs: 20, enrichStacktrace: async item => { await new Promise(resolve => { @@ -153,7 +153,7 @@ test("queue capacity and abort have structured outcomes and release capacity", a const ready = new Promise(resolve => { entered = resolve; }); - const ingester = createIngester({ + const ingester = createTelemetryStore({ maxPending: 1, enrichStacktrace: async (_item, { signal }) => { entered(); @@ -182,7 +182,7 @@ test("abort during post-ingestion retention replay restores the prior committed }); let calls = 0; let replaySignal: AbortSignal | undefined; - const ingester = createIngester({ + const ingester = createTelemetryStore({ retention: { maxEnvelopes: 1 }, enrichStacktrace: async (_event, { signal }) => { if (++calls === 1) return; @@ -207,7 +207,7 @@ test("abort during post-ingestion retention replay restores the prior committed }); test("byte retention evicts dependent data and delivery keys have the same lifetime", async () => { - const ingester = createIngester({ retention: { maxBytes: 100 } }); + const ingester = createTelemetryStore({ retention: { maxBytes: 100 } }); await ingester.ingestEnvelope(event("one"), { deliveryId: "one" }); await ingester.ingestEnvelope(event("two"), { deliveryId: "two" }); expect(ingester.getStats().retainedBytes).toBeLessThanOrEqual(100); @@ -217,7 +217,7 @@ test("byte retention evicts dependent data and delivery keys have the same lifet }); test("changes identify ingestion, eviction and reset while callbacks see committed reads", async () => { - const ingester = createIngester({ retention: { maxEnvelopes: 1 } }); + const ingester = createTelemetryStore({ retention: { maxEnvelopes: 1 } }); const changes: string[] = []; ingester.subscribe((snapshot, change) => { expect(snapshot).toBe(ingester.getSnapshot()); @@ -229,3 +229,169 @@ test("changes identify ingestion, eviction and reset while callbacks see committ expect(changes).toEqual(["ingest", "ingest", "reset"]); ingester.dispose(); }); + +test.each(["core", "node"] as const)( + "%s live retention preserves subscriptions and applies partial limits", + async runtime => { + const create = runtime === "core" ? createTelemetryStore : createTelemetryPipeline; + const client = create({ retention: { maxEnvelopes: 1 } }); + const changes: string[] = []; + const unsubscribe = client.subscribe((_snapshot, change) => changes.push(change.type)); + await client.ingestEnvelope(event("one")); + await client.setRetention({ maxEnvelopes: 3 }); + await client.ingestEnvelope(event("two")); + await client.ingestEnvelope(event("three")); + expect( + client + .getEvents() + .map(item => item.event_id) + .sort(), + ).toEqual(["one", "three", "two"]); + await client.setRetention({ maxEnvelopes: 2 }); + expect( + client + .getEvents() + .map(item => item.event_id) + .sort(), + ).toEqual(["three", "two"]); + expect(changes).toEqual(["ingest", "ingest", "ingest", "evict"]); + await client.setRetention({ maxBytes: 1 }); + expect(client.getEnvelopes()).toEqual([]); + await client.setRetention({ maxBytes: 10000 }); + await client.ingestEnvelope(event("four")); + await client.ingestEnvelope(event("five")); + await client.ingestEnvelope(event("six")); + expect(client.getEvents().map(item => item.event_id)).toEqual(["five", "six"]); + unsubscribe(); + client.dispose(); + }, +); + +test.each(["core", "node"] as const)( + "%s live age retention reschedules expiry and validates updates atomically", + async runtime => { + vi.useFakeTimers(); + const create = runtime === "core" ? createTelemetryStore : createTelemetryPipeline; + const client = create({ retention: { maxEnvelopes: 3, maxAgeMs: 1000 } }); + await client.ingestEnvelope(event("one")); + await client.ingestEnvelope(event("two")); + for (const update of [ + { maxEnvelopes: 1, maxBytes: 0 }, + { maxEnvelopes: 1, maxAgeMs: Number.NaN }, + { maxEnvelopes: 0 }, + ]) { + expect(() => client.setRetention(update)).toThrow(RangeError); + } + await client.prune(); + expect(client.getEnvelopes()).toHaveLength(2); + vi.setSystemTime(Date.now() + 100); + await client.setRetention({ maxAgeMs: 200 }); + expect(client.getEnvelopes()).toHaveLength(2); + await vi.advanceTimersByTimeAsync(101); + expect(client.getEnvelopes()).toEqual([]); + await client.ingestEnvelope(event("three")); + await client.setRetention({ maxEnvelopes: 2 }); + await vi.advanceTimersByTimeAsync(201); + expect(client.getEnvelopes()).toEqual([]); + client.dispose(); + await expect(client.setRetention({ maxEnvelopes: 0 })).resolves.toBeUndefined(); + }, +); + +test("Node retention waits for earlier payloads, preserves decoded history, and participates in drain", async () => { + let releaseFirst!: () => void; + let releaseLater!: () => void; + let enteredFirst!: () => void; + let enteredLater!: () => void; + const ready = new Promise(resolve => { + enteredFirst = resolve; + }); + const laterReady = new Promise(resolve => { + enteredLater = resolve; + }); + const client = createTelemetryPipeline({ + retention: { maxEnvelopes: 3 }, + enrichStacktrace: async item => { + if (item.event_id === "first") { + enteredFirst(); + await new Promise(resolve => { + releaseFirst = resolve; + }); + } else if (item.event_id === "later") { + enteredLater(); + await new Promise(resolve => { + releaseLater = resolve; + }); + } + }, + }); + const wire = (id: string) => `{}\n{"type":"event"}\n${JSON.stringify(error(id)[1][0][1])}\n`; + const initial = await client.ingestPayload({ body: wire("initial") }); + const changes: number[] = []; + client.subscribe(snapshot => changes.push(snapshot.envelopes.length)); + const first = client.ingestPayload({ body: wire("first") }); + await ready; + const update = client.setRetention({ maxEnvelopes: 1 }); + const drained = client.drain(); + const later = client.ingestPayload({ body: wire("later") }); + releaseFirst(); + await drained; + await update; + expect((await first).retained).toBe(true); + expect(changes.slice(0, 2)).toEqual([2, 1]); + expect(client.getEvents().map(item => item.event_id)).toEqual(["first"]); + if (initial.status !== "processed") throw new Error("Expected processed envelope"); + expect(client.getDecodedPayload(initial.envelopeId)?.body.toString()).toBe(wire("initial")); + await laterReady; + releaseLater(); + await later; + expect(client.getEvents().map(item => item.event_id)).toEqual(["later"]); + client.dispose(); +}); + +test("reset invalidates queued Node retention updates while completed limits persist", async () => { + const client = createTelemetryPipeline({ retention: { maxEnvelopes: 3 } }); + const update = client.setRetention({ maxEnvelopes: 1 }); + client.reset(); + await update; + await client.ingestEnvelopes([event("one"), event("two")]); + expect(client.getEnvelopes()).toHaveLength(2); + await client.setRetention({ maxEnvelopes: 1 }); + client.reset(); + await client.ingestEnvelopes([event("three"), event("four")]); + expect(client.getEvents().map(item => item.event_id)).toEqual(["four"]); + client.dispose(); +}); + +test.each(["core", "node"] as const)( + "%s retention update cannot reuse a prune barrier before later ingestion", + async runtime => { + let release!: () => void; + let entered!: () => void; + const ready = new Promise(resolve => { + entered = resolve; + }); + const create = runtime === "core" ? createTelemetryStore : createTelemetryPipeline; + const client = create({ + enrichStacktrace: async () => { + entered(); + await new Promise(resolve => { + release = resolve; + }); + }, + }); + const earlierPrune = client.prune(); + const ingestion = client.ingestEnvelope(error()); + let completed = false; + const update = client.setRetention({ maxEnvelopes: 1 }).then(() => { + completed = true; + }); + await earlierPrune; + await ready; + expect(completed).toBe(false); + release(); + await update; + expect((await ingestion).status).toBe("processed"); + client.dispose(); + }, +); diff --git a/packages/spotlight/src/telemetry/parity.spec.ts b/packages/spotlight/src/telemetry/parity.spec.ts index 75ba59149..f8e721c5d 100644 --- a/packages/spotlight/src/telemetry/parity.spec.ts +++ b/packages/spotlight/src/telemetry/parity.spec.ts @@ -2,14 +2,14 @@ import { readFileSync } from "node:fs"; import type { Envelope } from "@sentry/core"; import { expect, test } from "vitest"; -import { createIngester } from "../sdk/core.ts"; -import { createPipeline } from "../sdk/node.ts"; +import { createTelemetryStore } from "../sdk/core.ts"; +import { createTelemetryPipeline } from "../sdk/node.ts"; import useSentryStore from "../ui/telemetry/store/store"; -import { getAITraces, getQuerySummaries, getSpans } from "./selectors/index.ts"; +import { getAITraces, getDatabaseQuerySummaries, getSpans } from "./selectors/index.ts"; test("the UI store, browser processor, and Node SDK derive identical traces and queries", async () => { - const spotlight = createPipeline(); - const browser = createIngester(); + const spotlight = createTelemetryPipeline(); + const browser = createTelemetryStore(); const result = await spotlight.ingestPayload({ body: readFileSync(new URL("../../_fixtures/envelope_span_v2.txt", import.meta.url)), contentType: "application/x-sentry-envelope", @@ -23,15 +23,17 @@ test("the UI store, browser processor, and Node SDK derive identical traces and const uiTraces = useSentryStore.getState().getTraces(); expect(uiTraces).toEqual(spotlight.getTraces()); expect(browser.getTraces()).toEqual(spotlight.getTraces()); - expect(getQuerySummaries(getSpans(uiTraces))).toEqual(spotlight.getQuerySummaries({ includeSpans: true })); - expect(browser.getQuerySummaries()).toEqual(spotlight.getQuerySummaries()); + expect(getDatabaseQuerySummaries(getSpans(uiTraces))).toEqual( + spotlight.getDatabaseQuerySummaries({ includeSpans: true }), + ); + expect(browser.getDatabaseQuerySummaries()).toEqual(spotlight.getDatabaseQuerySummaries()); expect(JSON.stringify(envelope)).toBe(original); useSentryStore.getState().resetData(); }); test("the UI, browser processor, and Node SDK share log, profile, and AI processing", async () => { - const spotlight = createPipeline(); - const browser = createIngester(); + const spotlight = createTelemetryPipeline(); + const browser = createTelemetryStore(); useSentryStore.getState().resetData(); for (const file of ["continuous_profiling/transaction.txt", "continuous_profiling/profile_chunk_1.txt"]) { const result = await spotlight.ingestPayload({ diff --git a/packages/spotlight/src/telemetry/public.ts b/packages/spotlight/src/telemetry/public.ts index 29c59639f..7a012fad1 100644 --- a/packages/spotlight/src/telemetry/public.ts +++ b/packages/spotlight/src/telemetry/public.ts @@ -40,7 +40,7 @@ export type DeepReadonly = T extends (...args: never[]) => unknown /** A parsed Sentry wire envelope; input timestamps retain their wire units. */ // Item payloads remain unknown: forward-compatible wire records include span-v2 // containers that older @sentry/core payload unions cannot represent. -export type RawEnvelope = [Envelope[0], [EnvelopeItem[0], unknown][]]; +export type SentryEnvelope = [Envelope[0], [EnvelopeItem[0], unknown][]]; export type ProcessedEvent = DeepReadonly; export type ProcessedSpan = DeepReadonly; export type ProcessedTrace = DeepReadonly; @@ -64,7 +64,7 @@ export type RetentionOptions = { /** Serialized retained-envelope budget. Default: 64 MiB. */ maxBytes?: number; }; -export type IngesterOptions = { +export type TelemetryStoreOptions = { retention?: RetentionOptions; enrichStacktrace?: (event: SentryErrorEvent, context: { signal: AbortSignal }) => Promise; /** Maximum ingestion operations, including active work. Default: 64. Overflow returns ignored/queue_full. */ @@ -75,23 +75,25 @@ export type IngesterOptions = { enrichmentTimeoutMs?: number; onDiagnostic?: (diagnostic: Diagnostic) => void; }; -export type IngesterSnapshot = { +export type TelemetrySnapshot = { readonly revision: number; - readonly envelopes: readonly DeepReadonly[]; + readonly envelopes: readonly DeepReadonly[]; readonly events: readonly ProcessedEvent[]; readonly traces: readonly ProcessedTrace[]; readonly logs: readonly ProcessedLog[]; readonly sdks: readonly DeepReadonly[]; readonly profiles: ReadonlyMap; }; -export type Ingester = TelemetryReader & { +export type TelemetryStore = TelemetryReader & { ingestEnvelope: (input: unknown, options?: IngestOptions) => Promise; - ingestBatch: (inputs: Iterable, options?: BatchOptions) => Promise; + ingestEnvelopes: (inputs: Iterable, options?: BatchOptions) => Promise; /** Wait for operations submitted before this call; future operations are excluded. */ drain: () => Promise; getStats: () => IngestStats; - subscribe: (listener: (snapshot: IngesterSnapshot, change: ChangeMetadata) => void) => () => void; + subscribe: (listener: (snapshot: TelemetrySnapshot, change: ChangeMetadata) => void) => () => void; reset: () => void; + /** Update supplied limits, preserving omitted fields, then wait for retention eviction. */ + setRetention: (options: RetentionOptions) => Promise; prune: () => Promise; dispose: () => void; }; diff --git a/packages/spotlight/src/telemetry/reader.spec.ts b/packages/spotlight/src/telemetry/reader.spec.ts index 31f5bbb86..77a5a80b9 100644 --- a/packages/spotlight/src/telemetry/reader.spec.ts +++ b/packages/spotlight/src/telemetry/reader.spec.ts @@ -1,6 +1,6 @@ // @vitest-environment node import { expect, test } from "vitest"; -import type { IngesterSnapshot } from "./public.ts"; +import type { TelemetrySnapshot } from "./public.ts"; import { createTelemetryReader } from "./reader.ts"; import type { Span, Trace } from "./types.ts"; function fixture() { @@ -28,7 +28,7 @@ function fixture() { spanTree: spans, profileGrafted: false, }; - const snapshot: IngesterSnapshot = { + const snapshot: TelemetrySnapshot = { revision: 1, envelopes: [], events: [ @@ -75,10 +75,10 @@ test("invalid getter filters throw", () => { test("summaries default to compact and preserve trace-less legacy spans", () => { const { snapshot } = fixture(); const reader = createTelemetryReader(() => snapshot); - expect(reader.getQuerySummaries({ traceId: "trace", toMs: 20 })).toEqual([ + expect(reader.getDatabaseQuerySummaries({ traceId: "trace", toMs: 20 })).toEqual([ { description: "SELECT 1", count: 2, totalDurationMs: 25, averageDurationMs: 12.5 }, ]); - expect(reader.getQuerySummaries({ includeSpans: true })[0].spans).toHaveLength(2); + expect(reader.getDatabaseQuerySummaries({ includeSpans: true })[0].spans).toHaveLength(2); expect(reader.getResourceSummaries()).toEqual([ { description: "script.js", count: 1, totalDurationMs: 5, averageDurationMs: 5, averageEncodedSizeBytes: 40 }, ]); @@ -87,11 +87,11 @@ test("summaries default to compact and preserve trace-less legacy spans", () => test("canonical filters share cached identity until revision changes", () => { let { snapshot } = fixture(); const reader = createTelemetryReader(() => snapshot); - const first = reader.getQuerySummaries({ traceId: "trace", fromMs: 10 }); - expect(reader.getQuerySummaries({ fromMs: 10, traceId: "trace" })).toBe(first); + const first = reader.getDatabaseQuerySummaries({ traceId: "trace", fromMs: 10 }); + expect(reader.getDatabaseQuerySummaries({ fromMs: 10, traceId: "trace" })).toBe(first); expect(reader.getEvents()).toBe(reader.getEvents({})); snapshot = { ...snapshot, revision: 2 }; - expect(reader.getQuerySummaries({ fromMs: 10, traceId: "trace" })).not.toBe(first); + expect(reader.getDatabaseQuerySummaries({ fromMs: 10, traceId: "trace" })).not.toBe(first); }); test("page cursors bind collection, canonical query and revision", () => { let { snapshot } = fixture(); @@ -143,7 +143,7 @@ test("explicit default filter values share cached results", () => { const { snapshot } = fixture(); const reader = createTelemetryReader(() => snapshot); expect(reader.getEvents({ order: "asc" })).toBe(reader.getEvents()); - expect(reader.getQuerySummaries({ includeSpans: false })).toBe(reader.getQuerySummaries()); + expect(reader.getDatabaseQuerySummaries({ includeSpans: false })).toBe(reader.getDatabaseQuerySummaries()); }); test("web vitals apply limit after selecting measurements", () => { diff --git a/packages/spotlight/src/telemetry/reader.ts b/packages/spotlight/src/telemetry/reader.ts index 78089c4a6..34b41b6d2 100644 --- a/packages/spotlight/src/telemetry/reader.ts +++ b/packages/spotlight/src/telemetry/reader.ts @@ -2,7 +2,6 @@ import { readonlyCopy } from "./internal/readonly.ts"; import type { AITrace, DeepReadonly, - IngesterSnapshot, ProcessedEvent, ProcessedLog, ProcessedProfile, @@ -11,11 +10,12 @@ import type { ProfileFunctionSummary, QuerySummary, ResourceSummary, + TelemetrySnapshot, } from "./public.ts"; import { getAITraces, + getDatabaseQuerySummaries, getProfileFunctionSummaries, - getQuerySummaries, getResourceSummaries, getWebVitals, } from "./selectors/index.ts"; @@ -58,9 +58,9 @@ export type ReadPageResult = readonly message: string; }; export type TelemetryReader = { - getSnapshot(): IngesterSnapshot; - getEnvelope(id: string): IngesterSnapshot["envelopes"][number] | undefined; - getEnvelopes(): IngesterSnapshot["envelopes"]; + getSnapshot(): TelemetrySnapshot; + getEnvelope(id: string): TelemetrySnapshot["envelopes"][number] | undefined; + getEnvelopes(): TelemetrySnapshot["envelopes"]; getEvent(id: string): ProcessedEvent | undefined; getEvents(filter?: EventReadFilter): readonly ProcessedEvent[]; getErrors(filter?: EventReadFilter): readonly DeepReadonly[]; @@ -71,9 +71,9 @@ export type TelemetryReader = { getLogs(filter?: TraceReadFilter): readonly ProcessedLog[]; getProfile(traceId: string): ProcessedProfile | undefined; getProfileFunctionSummaries(filter?: TraceReadFilter): readonly ProfileFunctionSummary[]; - getQuerySummaries(filter: DatabaseReadFilter & { includeSpans: true }): readonly QuerySummary[]; - getQuerySummaries(filter?: DatabaseReadFilter & { includeSpans?: false }): readonly CompactQuerySummary[]; - getQuerySummaries( + getDatabaseQuerySummaries(filter: DatabaseReadFilter & { includeSpans: true }): readonly QuerySummary[]; + getDatabaseQuerySummaries(filter?: DatabaseReadFilter & { includeSpans?: false }): readonly CompactQuerySummary[]; + getDatabaseQuerySummaries( filter?: DatabaseReadFilter, ): readonly (CompactQuerySummary & { readonly spans?: QuerySummary["spans"] })[]; getResourceSummaries(filter: SummaryReadFilter & { includeSpans: true }): readonly ResourceSummary[]; @@ -140,10 +140,10 @@ function named(items: readonly T[], filter: TelemetryReadFilter, name: (item: } /** A portable read-only dependency. The current revision retains at most 256 recently used reads. */ -export function createTelemetryReader(getSnapshot: () => IngesterSnapshot): TelemetryReader { +export function createTelemetryReader(getSnapshot: () => TelemetrySnapshot): TelemetryReader { let revision = -1; const cache = new Map(); - function read(name: string, filter: object, derive: (snapshot: IngesterSnapshot) => T): T { + function read(name: string, filter: object, derive: (snapshot: TelemetrySnapshot) => T): T { const snapshot = getSnapshot(); if (revision !== snapshot.revision) { cache.clear(); @@ -161,12 +161,12 @@ export function createTelemetryReader(getSnapshot: () => IngesterSnapshot): Tele if (cache.size > 256) cache.delete(cache.keys().next().value!); return value; } - const traces = (snapshot: IngesterSnapshot, filter: TraceReadFilter) => + const traces = (snapshot: TelemetrySnapshot, filter: TraceReadFilter) => snapshot.traces.filter(trace => filter.traceId === undefined || trace.trace_id === filter.traceId); // Legacy transaction child spans need not carry trace_id. Scope their owning trace first. - const spans = (snapshot: IngesterSnapshot, filter: TraceReadFilter) => + const spans = (snapshot: TelemetrySnapshot, filter: TraceReadFilter) => traces(snapshot, filter).flatMap(trace => Array.from(trace.spans.values())); - function events(snapshot: IngesterSnapshot, filter: EventReadFilter) { + function events(snapshot: TelemetrySnapshot, filter: EventReadFilter) { return snapshot.events.filter( event => (filter.traceId === undefined || @@ -261,7 +261,7 @@ export function createTelemetryReader(getSnapshot: () => IngesterSnapshot): Tele ), ); }, - getQuerySummaries: ((filter: DatabaseReadFilter = {}) => { + getDatabaseQuerySummaries: ((filter: DatabaseReadFilter = {}) => { validate(filter); if (filter.description !== undefined && typeof filter.description !== "string") throw new RangeError("description must be a string"); @@ -270,7 +270,7 @@ export function createTelemetryReader(getSnapshot: () => IngesterSnapshot): Tele return read("queries", filter, snapshot => readonlyCopy( named( - getQuerySummaries( + getDatabaseQuerySummaries( ordered( spans(snapshot, filter), { fromMs: filter.fromMs, toMs: filter.toMs, order: filter.order }, @@ -287,7 +287,7 @@ export function createTelemetryReader(getSnapshot: () => IngesterSnapshot): Tele ), ), ); - }) as TelemetryReader["getQuerySummaries"], + }) as TelemetryReader["getDatabaseQuerySummaries"], getResourceSummaries: ((filter: SummaryReadFilter = {}) => { validate(filter); if (filter.includeSpans !== undefined && typeof filter.includeSpans !== "boolean") diff --git a/packages/spotlight/src/telemetry/selectors/database.ts b/packages/spotlight/src/telemetry/selectors/database.ts index 5007d4214..9993f3570 100644 --- a/packages/spotlight/src/telemetry/selectors/database.ts +++ b/packages/spotlight/src/telemetry/selectors/database.ts @@ -6,7 +6,7 @@ export type QueryFilter = SpanFilter & { description?: string }; export type QuerySummary = SpanSummary; /** Inputs and durations use normalized milliseconds. Presentation sorting belongs to callers. */ -export function getQuerySummaries(spans: readonly Span[], filter: QueryFilter = {}): QuerySummary[] { +export function getDatabaseQuerySummaries(spans: readonly Span[], filter: QueryFilter = {}): QuerySummary[] { return summarizeSpans( spans.filter( span => DB_SPAN_REGEX.test(span.op ?? "") && (filter.traceId === undefined || span.trace_id === filter.traceId), diff --git a/packages/spotlight/src/telemetry/selectors/index.ts b/packages/spotlight/src/telemetry/selectors/index.ts index 8e5913480..e57930aef 100644 --- a/packages/spotlight/src/telemetry/selectors/index.ts +++ b/packages/spotlight/src/telemetry/selectors/index.ts @@ -1,5 +1,5 @@ export { getAIRootSpans, getAITraces } from "./ai.ts"; -export { DB_SPAN_REGEX, getQuerySummaries, type QueryFilter, type QuerySummary } from "./database.ts"; +export { DB_SPAN_REGEX, getDatabaseQuerySummaries, type QueryFilter, type QuerySummary } from "./database.ts"; export { getProfileFunctionSummaries, type ProfileFunctionSummary } from "./profiles.ts"; export { getResourceSummaries, type ResourceSummary } from "./resources.ts"; export { getSpans, type SpanFilter } from "./spans.ts"; diff --git a/packages/spotlight/src/telemetry/selectors/selectors.test.ts b/packages/spotlight/src/telemetry/selectors/selectors.test.ts index 88646304b..974debe7f 100644 --- a/packages/spotlight/src/telemetry/selectors/selectors.test.ts +++ b/packages/spotlight/src/telemetry/selectors/selectors.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { SentryProcessedProfile, Span } from "../types.ts"; -import { getQuerySummaries } from "./database.ts"; +import { getDatabaseQuerySummaries } from "./database.ts"; import { getProfileFunctionSummaries } from "./profiles.ts"; import { getResourceSummaries } from "./resources.ts"; @@ -36,7 +36,7 @@ describe("domain summaries", () => { it("groups database spans using normalized millisecond fields and excludes invalid inputs", () => { const first = span(); const second = span({ span_id: "second", description: "SELECT 1", timestamp: 1030 }); - const result = getQuerySummaries([ + const result = getDatabaseQuerySummaries([ first, span({ op: "http.client" }), second, @@ -55,9 +55,9 @@ describe("domain summaries", () => { }, ]); expect(first.description).toBe(" SELECT 1 "); - expect(getQuerySummaries([first], { description: " SELECT 1 " })).toHaveLength(1); - expect(getQuerySummaries([first], { description: "SELECT 2" })).toEqual([]); - expect(getQuerySummaries([first], { traceId: "trace-b" })).toEqual([]); + expect(getDatabaseQuerySummaries([first], { description: " SELECT 1 " })).toHaveLength(1); + expect(getDatabaseQuerySummaries([first], { description: "SELECT 2" })).toEqual([]); + expect(getDatabaseQuerySummaries([first], { traceId: "trace-b" })).toEqual([]); }); it("reports resource size in bytes and retains member spans once", () => { diff --git a/packages/spotlight/src/telemetry/serialization.spec.ts b/packages/spotlight/src/telemetry/serialization.spec.ts index 3b8d0cbd9..7d7677aeb 100644 --- a/packages/spotlight/src/telemetry/serialization.spec.ts +++ b/packages/spotlight/src/telemetry/serialization.spec.ts @@ -82,8 +82,8 @@ test("span tree and transaction copies merge their child references without drop test("valid continuous profiles and streamed plus legacy traces serialize", async () => { const { readFileSync } = await import("node:fs"); - const { createPipeline } = await import("../sdk/node.ts"); - const pipeline = createPipeline(); + const { createTelemetryPipeline } = await import("../sdk/node.ts"); + const pipeline = createTelemetryPipeline(); try { for (const file of [ "continuous_profiling/transaction.txt", @@ -119,8 +119,8 @@ test("a wide span graph does not use call-argument expansion", () => { }); test("canonical spans serialize after streamed updates supersede transaction span copies", async () => { - const { createIngester } = await import("./index.ts"); - const client = createIngester(); + const { createTelemetryStore } = await import("./index.ts"); + const client = createTelemetryStore(); try { expect( ( diff --git a/packages/spotlight/src/telemetry/serialization.ts b/packages/spotlight/src/telemetry/serialization.ts index 971aa30ca..74e320772 100644 --- a/packages/spotlight/src/telemetry/serialization.ts +++ b/packages/spotlight/src/telemetry/serialization.ts @@ -1,9 +1,9 @@ import type { DeepReadonly, - IngesterSnapshot, ProcessedSpan, ProcessedTrace, ProfileFunctionSummary, + TelemetrySnapshot, } from "./public.ts"; import type { CompactQuerySummary, CompactResourceSummary } from "./reader.ts"; @@ -125,7 +125,7 @@ export function serializeTrace(trace: ProcessedTrace): SerializedTrace { } /** A JSON-ready DTO: Map entries and Set values become arrays; no custom JSON replacer is needed. */ -export function serializeSnapshot(snapshot: IngesterSnapshot): SerializedSnapshot { +export function serializeSnapshot(snapshot: TelemetrySnapshot): SerializedSnapshot { return { schemaVersion: 1, revision: snapshot.revision, diff --git a/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx b/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx index 88f3c8c0e..6405c8cd3 100644 --- a/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx +++ b/packages/spotlight/src/ui/telemetry/components/events/EventContexts.test.tsx @@ -1,12 +1,12 @@ import { cleanup, render, screen } from "@testing-library/react"; import { afterEach, expect, test } from "vitest"; -import { createIngester } from "../../../../telemetry/index.ts"; +import { createTelemetryStore } from "../../../../telemetry/index.ts"; import EventContexts from "./EventContexts"; afterEach(cleanup); test("renders modules alongside extra data without mutating the SDK event", async () => { - const telemetry = createIngester(); + const telemetry = createTelemetryStore(); await telemetry.ingestEnvelope([ {}, [ diff --git a/packages/spotlight/src/ui/telemetry/components/insights/Queries.tsx b/packages/spotlight/src/ui/telemetry/components/insights/Queries.tsx index 384718f80..5f9a1116e 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/Queries.tsx +++ b/packages/spotlight/src/ui/telemetry/components/insights/Queries.tsx @@ -9,7 +9,7 @@ import { getFormattedDuration, getSpanDurationClassName } from "@spotlight/ui/te import Table from "@spotlight/ui/ui/table"; import { useMemo } from "react"; import { useNavigate } from "react-router-dom"; -import { type QuerySummary, getQuerySummaries } from "../../../../telemetry/selectors/database.ts"; +import { type QuerySummary, getDatabaseQuerySummaries } from "../../../../telemetry/selectors/database.ts"; import EmptyState from "../shared/EmptyState"; type QueryInfo = QuerySummary; @@ -33,7 +33,9 @@ const Queries = () => { const queriesData: QueryInfo[] = useMemo(() => { const compareQueryInfo = COMPARATORS[sort.active] || COMPARATORS[QUERIES_SORT_KEYS.totalTime]; - return getQuerySummaries(allSpans).sort((a, b) => (sort.asc ? compareQueryInfo(a, b) : compareQueryInfo(b, a))); + return getDatabaseQuerySummaries(allSpans).sort((a, b) => + sort.asc ? compareQueryInfo(a, b) : compareQueryInfo(b, a), + ); }, [allSpans, sort]); const maxTime = Math.max(...queriesData.map(query => query.totalDurationMs)); diff --git a/packages/spotlight/src/ui/telemetry/components/insights/QuerySummary.tsx b/packages/spotlight/src/ui/telemetry/components/insights/QuerySummary.tsx index 88c896b81..9a4134146 100644 --- a/packages/spotlight/src/ui/telemetry/components/insights/QuerySummary.tsx +++ b/packages/spotlight/src/ui/telemetry/components/insights/QuerySummary.tsx @@ -6,7 +6,7 @@ import Breadcrumbs from "@spotlight/ui/ui/breadcrumbs"; import Table from "@spotlight/ui/ui/table"; import { useMemo } from "react"; import { Link, useParams } from "react-router-dom"; -import { getQuerySummaries } from "../../../../telemetry/selectors/database.ts"; +import { getDatabaseQuerySummaries } from "../../../../telemetry/selectors/database.ts"; import { QUERY_SUMMARY_HEADERS, QUERY_SUMMARY_SORT_KEYS } from "../../constants"; import { useSentrySpans } from "../../data/useSentrySpans"; import useSort from "../../hooks/useSort"; @@ -49,7 +49,7 @@ const QuerySummary = () => { const spans = allSpans; const compareSpanInfo = COMPARATORS[sort.active] || COMPARATORS[QUERY_SUMMARY_SORT_KEYS.totalTime]; - return (getQuerySummaries(spans, { description: decodedType })[0]?.spans ?? []).sort((a, b) => + return (getDatabaseQuerySummaries(spans, { description: decodedType })[0]?.spans ?? []).sort((a, b) => sort.asc ? compareSpanInfo(a, b) : compareSpanInfo(b, a), ); }, [allSpans, sort, decodedType]); diff --git a/packages/spotlight/src/ui/telemetry/store/store.ts b/packages/spotlight/src/ui/telemetry/store/store.ts index 2d0651d9c..5d738970f 100644 --- a/packages/spotlight/src/ui/telemetry/store/store.ts +++ b/packages/spotlight/src/ui/telemetry/store/store.ts @@ -2,8 +2,8 @@ import type { Envelope } from "@sentry/core"; import { CONTEXT_LINES_ENDPOINT } from "@spotlight/shared/constants.ts"; import { DEFAULT_SIDECAR_URL } from "@spotlight/ui/constants"; import { useStore } from "zustand"; -import { createIngester } from "../../../sdk/core.ts"; -import type { IngestResult, IngesterSnapshot } from "../../../sdk/core.ts"; +import { createTelemetryStore } from "../../../sdk/core.ts"; +import type { IngestResult, TelemetrySnapshot } from "../../../sdk/core.ts"; import type { SentryStore, SentryStoreState, Subscription } from "../../../telemetry/store/types.ts"; import { getNativeFetchImplementation } from "../utils/fetch"; @@ -12,7 +12,7 @@ export function setTelemetrySidecarUrl(url: string) { contextLinesProvider = new URL(CONTEXT_LINES_ENDPOINT, url).href; } -const spotlight = createIngester({ +const spotlight = createTelemetryStore({ enrichStacktrace: async (event, { signal }) => { await Promise.all( (event.exception?.values ?? []).map(async exception => { @@ -91,8 +91,8 @@ const actions = { }; }, }; -function view(snapshot: IngesterSnapshot): UiState { - const logsByTraceId = new Map>(); +function view(snapshot: TelemetrySnapshot): UiState { + const logsByTraceId = new Map>(); for (const log of snapshot.logs) { if (!log.trace_id) continue; const group = logsByTraceId.get(log.trace_id) ?? new Set(); diff --git a/packages/spotlight/tests/types/sdk-core.ts b/packages/spotlight/tests/types/sdk-core.ts index 210da8a43..720e5119f 100644 --- a/packages/spotlight/tests/types/sdk-core.ts +++ b/packages/spotlight/tests/types/sdk-core.ts @@ -4,15 +4,15 @@ import { type QuerySummary, SDK_CAPABILITIES, type TelemetryReader, - createIngester, createTelemetryReader, + createTelemetryStore, serializeSnapshot, serializeSummaries, serializeTrace, } from "@spotlightjs/spotlight/sdk/core"; -const telemetry = createIngester(); -const queries: readonly QuerySummary[] = telemetry.getQuerySummaries({ traceId: "trace", includeSpans: true }); +const telemetry = createTelemetryStore(); +const queries: readonly QuerySummary[] = telemetry.getDatabaseQuerySummaries({ traceId: "trace", includeSpans: true }); const events: readonly ProcessedEvent[] = telemetry.getEvents({ traceId: "trace", spanId: "span" }); const envelope = telemetry.getEnvelope("envelope"); if (envelope) { @@ -41,7 +41,7 @@ void queries[0].averageDurationMs; const reader: TelemetryReader = createTelemetryReader(telemetry.getSnapshot); // @ts-expect-error Read-only consumers cannot ingest. reader.ingestEnvelope; -const compact = reader.getQuerySummaries(); +const compact = reader.getDatabaseQuerySummaries(); // @ts-expect-error Full spans require includeSpans: true. const full: readonly QuerySummary[] = compact; void full; @@ -68,7 +68,7 @@ serializeSummaries(queries); void version; void unit; async function receipts() { - const results: readonly IngestResult[] = await telemetry.ingestBatch([null], { + const results: readonly IngestResult[] = await telemetry.ingestEnvelopes([null], { signal: new AbortController().signal, }); const receipt = await telemetry.ingestEnvelope(null, { deliveryId: "retry" }); @@ -88,6 +88,8 @@ async function receipts() { void severity; void index; } + const resized: Promise = telemetry.setRetention({ maxEnvelopes: 10 }); + await resized; await telemetry.drain(); void results; void retained; diff --git a/packages/spotlight/tests/types/sdk-node.ts b/packages/spotlight/tests/types/sdk-node.ts index 4605190f3..73ecfeecf 100644 --- a/packages/spotlight/tests/types/sdk-node.ts +++ b/packages/spotlight/tests/types/sdk-node.ts @@ -1,27 +1,32 @@ import type { IngestResult, TelemetryReader } from "@spotlightjs/spotlight/sdk/core"; -import { type PipelineInput, createPipeline } from "@spotlightjs/spotlight/sdk/node"; +import { type TelemetryPayload, createTelemetryPipeline } from "@spotlightjs/spotlight/sdk/node"; -const pipeline = createPipeline({ +const pipeline = createTelemetryPipeline({ maxPending: 4, maxPayloadBytes: 1024, maxDecodedBytes: 2048, - maxRawBytes: 4096, - rawBufferSize: 3, + decodedHistory: { maxBytes: 4096, maxEntries: 3 }, retention: { maxEnvelopes: 10, maxBytes: 8192, maxAgeMs: 1000 }, enrichStacktrace: async (_event, { signal }) => { signal.throwIfAborted(); }, }); const reader: TelemetryReader = pipeline; -const input: PipelineInput = { body: new Uint8Array(), deliveryId: "request-1", signal: new AbortController().signal }; +const input: TelemetryPayload = { + body: new Uint8Array(), + deliveryId: "request-1", + signal: new AbortController().signal, +}; const receipt: Promise = pipeline.ingestPayload(input); -const batch: Promise = pipeline.ingestPayloadBatch([input]); +const batch: Promise = pipeline.ingestPayloads([input]); const decoded = pipeline.getDecodedPayload("envelope"); const body: Uint8Array | undefined = decoded?.body; // @ts-expect-error Serialized payload ingestion has an explicit name. pipeline.ingest; // @ts-expect-error Decoded history has an explicit name. pipeline.getRawPayload; +const resized: Promise = pipeline.setRetention({ maxEnvelopes: 10, maxAgeMs: 1000 }); +void resized; void reader; void receipt; void batch; From 487d3365770833bcb91197a0ecec5d0486e918b1 Mon Sep 17 00:00:00 2001 From: mathuraditya724 Date: Wed, 28 Oct 2026 15:29:28 +0530 Subject: [PATCH 7/7] fix(sdk): Preserve telemetry across runtime and trace edge cases Accept cross-realm byte inputs, preserve log-only traces and resource size metadata, and keep self-parented spans visible. Update the executable examples to the telemetry naming. Co-Authored-By: Codex --- packages/spotlight/README.md | 4 +- packages/spotlight/examples/sdk/frontend.ts | 8 +- packages/spotlight/examples/sdk/lifecycle.ts | 20 ++-- packages/spotlight/examples/sdk/react.tsx | 2 +- packages/spotlight/examples/sdk/server.ts | 9 +- packages/spotlight/src/sdk/node.spec.ts | 25 +++++ packages/spotlight/src/sdk/node.ts | 5 +- .../src/telemetry/log-traces.spec.ts | 100 +++++++++++++++++ .../spotlight/src/telemetry/reader.spec.ts | 105 +++++++++++++++--- .../src/telemetry/selectors/resources.ts | 45 ++++++-- .../src/telemetry/selectors/selectors.test.ts | 70 ++++++++++-- .../src/telemetry/store/slices/eventsSlice.ts | 32 +++++- .../spotlight/src/telemetry/topology.spec.ts | 30 +++++ .../spotlight/src/telemetry/utils/traces.ts | 4 +- 14 files changed, 402 insertions(+), 57 deletions(-) create mode 100644 packages/spotlight/src/telemetry/log-traces.spec.ts create mode 100644 packages/spotlight/src/telemetry/topology.spec.ts diff --git a/packages/spotlight/README.md b/packages/spotlight/README.md index 80a079d15..54d0ea934 100644 --- a/packages/spotlight/README.md +++ b/packages/spotlight/README.md @@ -140,7 +140,9 @@ Event reads accept `traceId` and `spanId`; trace/log/span/profile/summary reads accept `traceId`; query summaries also accept `description`. The time/order/limit filters apply to filtered collection reads, not `getEnvelopes()` or `getSdks()`. Summary fields name units explicitly, including `totalDurationMs`, -`averageDurationMs`, and `averageEncodedSizeBytes`. Stored envelopes retain wire +`averageDurationMs`, and `averageEncodedSizeBytes`. Resource summaries also expose optional `averageSizeBytes`, averaging known finite, nonnegative response sizes using encoded length, body size, and decoded length fallbacks. Unknown sizes are excluded; `averageEncodedSizeBytes` remains the encoded-specific field. + +Stored envelopes retain wire timestamp units. Caller input is copied; read results are readonly. `reset()` clears captured data and decoded history, keeps subscriptions, and diff --git a/packages/spotlight/examples/sdk/frontend.ts b/packages/spotlight/examples/sdk/frontend.ts index b2ba9d39d..5b626e472 100644 --- a/packages/spotlight/examples/sdk/frontend.ts +++ b/packages/spotlight/examples/sdk/frontend.ts @@ -2,11 +2,11 @@ import { type IngestOptions, createTelemetryStore } from "@spotlightjs/spotlight /** Keep one instance for the lifetime of the telemetry view. */ export function createFrontendTelemetry() { - const ingester = createTelemetryStore({ retention: { maxEnvelopes: 500 } }); + const telemetry = createTelemetryStore({ retention: { maxEnvelopes: 500 } }); return { - ingester, + telemetry, // Pass the parsed envelopes received over SSE, WebSocket, or your own transport. - receiveEnvelope: (envelope: unknown, options?: IngestOptions) => ingester.ingestEnvelope(envelope, options), - disconnect: ingester.dispose, + receiveEnvelope: (envelope: unknown, options?: IngestOptions) => telemetry.ingestEnvelope(envelope, options), + disconnect: telemetry.dispose, }; } diff --git a/packages/spotlight/examples/sdk/lifecycle.ts b/packages/spotlight/examples/sdk/lifecycle.ts index 50366af29..16cf8a79c 100644 --- a/packages/spotlight/examples/sdk/lifecycle.ts +++ b/packages/spotlight/examples/sdk/lifecycle.ts @@ -31,7 +31,7 @@ function check(condition: unknown, message: string): asserts condition { if (!condition) throw new Error(message); } -const { ingester, receiveEnvelope, disconnect } = createFrontendTelemetry(); +const { telemetry, receiveEnvelope, disconnect } = createFrontendTelemetry(); const pipeline = createTelemetryPipeline(); try { const input = envelope("one"); @@ -41,7 +41,7 @@ try { check(retry.status === "ignored" && retry.reason === "duplicate", "Retained delivery retry"); // Each input gets a receipt. A malformed middle input does not roll back valid neighbors. - const batch = await ingester.ingestEnvelopes([envelope("two"), null, envelope("three")]); + const batch = await telemetry.ingestEnvelopes([envelope("two"), null, envelope("three")]); check( batch[0].status === "processed" && batch[1].status === "invalid" && batch[2].status === "processed", "Partial batch", @@ -53,20 +53,20 @@ try { } const controller = new AbortController(); controller.abort(); - const cancelled = await ingester.ingestEnvelope(envelope("cancelled"), { signal: controller.signal }); + const cancelled = await telemetry.ingestEnvelope(envelope("cancelled"), { signal: controller.signal }); check(cancelled.status === "ignored" && cancelled.reason === "aborted", "Cancellation receipt"); - await ingester.drain(); // A barrier for work already submitted, not future submissions. - check(ingester.getStats().pending === 0, "Drained work"); + await telemetry.drain(); // A barrier for work already submitted, not future submissions. + check(telemetry.getStats().pending === 0, "Drained work"); - const reader = inspectTrace(ingester, "example-trace"); + const reader = inspectTrace(telemetry, "example-trace"); check(JSON.parse(reader.snapshotJson).schemaVersion === 1, "JSON schema"); check(JSON.parse(reader.profileJson).schemaVersion === 1, "Summary JSON schema"); - const page = ingester.readPage("events", { limit: 1 }); + const page = telemetry.readPage("events", { limit: 1 }); check(page.status === "ok" && page.nextCursor, "Bounded page"); await receiveEnvelope(envelope("four")); - const stale = ingester.readPage("events", { limit: 1, cursor: page.nextCursor }); + const stale = telemetry.readPage("events", { limit: 1, cursor: page.nextCursor }); check(stale.status === "invalid" && stale.code === "stale_cursor", "Revision-bound cursor"); - check(readEventIds(ingester).status === "ok", "Reader-only pagination"); + check(readEventIds(telemetry).status === "ok", "Reader-only pagination"); const wire = (id: string) => `{}\n${JSON.stringify({ type: "span" })}\n${JSON.stringify(envelope(id)[1][0][1])}`; const payloads = await pipeline.ingestPayloads([{ body: wire("node-one") }, { body: "invalid" }]); @@ -75,7 +75,7 @@ try { check(pipeline.getDecodedPayload(payloads[0].envelopeId)?.body.length, "Copied decoded payload"); } await pipeline.drain(); - console.log(SDK_CAPABILITIES.timestampUnit, first.counts, ingester.getStats()); + console.log(SDK_CAPABILITIES.timestampUnit, first.counts, telemetry.getStats()); } finally { pipeline.dispose(); disconnect(); diff --git a/packages/spotlight/examples/sdk/react.tsx b/packages/spotlight/examples/sdk/react.tsx index 138c3dfc9..11631808b 100644 --- a/packages/spotlight/examples/sdk/react.tsx +++ b/packages/spotlight/examples/sdk/react.tsx @@ -7,7 +7,7 @@ type Props = { traceId?: string; }; -/** Pass reader={ingester} subscribe={ingester.subscribe}; the view cannot ingest or reset. */ +/** Pass reader={telemetry} subscribe={telemetry.subscribe}; the view cannot ingest or reset. */ export function QueryList({ reader, subscribe, traceId }: Props) { useSyncExternalStore(subscribe, reader.getSnapshot, reader.getSnapshot); const queries = reader.getDatabaseQuerySummaries({ traceId }); diff --git a/packages/spotlight/examples/sdk/server.ts b/packages/spotlight/examples/sdk/server.ts index 95f817b5f..fb69800b2 100644 --- a/packages/spotlight/examples/sdk/server.ts +++ b/packages/spotlight/examples/sdk/server.ts @@ -40,7 +40,14 @@ export function createTelemetryServer() { signal: controller.signal, }); response.off("close", abort); - response.statusCode = result.status === "invalid" ? 400 : 202; + response.statusCode = + result.status === "invalid" + ? 400 + : result.status === "ignored" && result.reason === "unsupported_content_type" + ? 415 + : result.status === "ignored" && ["queue_full", "reset", "disposed", "aborted"].includes(result.reason) + ? 503 + : 202; response.end(JSON.stringify(result)); return; } diff --git a/packages/spotlight/src/sdk/node.spec.ts b/packages/spotlight/src/sdk/node.spec.ts index 5bcd9f043..4e40dc4fe 100644 --- a/packages/spotlight/src/sdk/node.spec.ts +++ b/packages/spotlight/src/sdk/node.spec.ts @@ -1,5 +1,6 @@ // @vitest-environment node import { readFileSync } from "node:fs"; +import { runInNewContext } from "node:vm"; import { brotliCompressSync, deflateSync, gzipSync } from "node:zlib"; import { describe, expect, test, vi } from "vitest"; import { createTelemetryPipeline } from "./node.ts"; @@ -763,3 +764,27 @@ test("diagnostic callbacks cannot bypass shared admission during core validation expect(await Promise.all(later)).toMatchObject([{ status: "ignored", reason: "queue_full" }]); pipeline.dispose(); }); + +test("accepts Uint8Array bodies created in another realm", async () => { + const body: Uint8Array = runInNewContext("Uint8Array.from(bytes)", { bytes: [...fixture] }); + expect(body instanceof Uint8Array).toBe(false); + const telemetry = createTelemetryPipeline(); + try { + expect((await telemetry.ingestPayload({ body })).status).toBe("processed"); + expect(telemetry.getTraces().length).toBeGreaterThan(0); + } finally { + telemetry.dispose(); + } +}); + +test("generated payload identities are canonical UUID strings", async () => { + const telemetry = createTelemetryPipeline(); + try { + const result = await telemetry.ingestPayload({ body: fixture }); + expect(result.status).toBe("processed"); + if (result.status !== "processed") throw new Error("Expected processed envelope"); + expect(result.envelopeId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + } finally { + telemetry.dispose(); + } +}); diff --git a/packages/spotlight/src/sdk/node.ts b/packages/spotlight/src/sdk/node.ts index 4a9e9e123..5da755bbb 100644 --- a/packages/spotlight/src/sdk/node.ts +++ b/packages/spotlight/src/sdk/node.ts @@ -1,4 +1,5 @@ import { Buffer } from "node:buffer"; +import { types as nodeTypes } from "node:util"; import { brotliDecompress, gunzip, inflate } from "node:zlib"; import type { Envelope } from "@sentry/core"; import type { BatchOptions, DiagnosticCode, IgnoreReason, IngestOptions } from "../telemetry/contracts.ts"; @@ -179,7 +180,7 @@ export function createTelemetryPipeline(options: TelemetryPipelineOptions = {}): } if (!envelope || parseFailed) return invalid("invalid_envelope", "Malformed serialized Sentry envelope"); // Normalize the codec's legacy UUID object before staging; the core publishes this exact string ID. - const envelopeId = input.deliveryId ?? String(envelope[0].__spotlight_envelope_id); + const envelopeId = input.deliveryId ?? envelope[0].__spotlight_envelope_id.toString(); const parsedEnvelope = envelope as Envelope; parsedEnvelope[0].__spotlight_envelope_id = envelopeId; const payload = { body, contentType }; @@ -261,7 +262,7 @@ export function createTelemetryPipeline(options: TelemetryPipelineOptions = {}): type PreparedPayload = { input: TelemetryPayload; body: Buffer; contentType: string; encoding?: string }; function preparePayload(input: TelemetryPayload): PreparedPayload | IngestResult { - if (!input || (typeof input.body !== "string" && !(input.body instanceof Uint8Array))) { + if (!input || (typeof input.body !== "string" && !nodeTypes.isUint8Array(input.body))) { return invalid("invalid_body", "Envelope body must be a string or Uint8Array"); } if (input.signal?.aborted) return ignored("aborted"); diff --git a/packages/spotlight/src/telemetry/log-traces.spec.ts b/packages/spotlight/src/telemetry/log-traces.spec.ts new file mode 100644 index 000000000..7f274e077 --- /dev/null +++ b/packages/spotlight/src/telemetry/log-traces.spec.ts @@ -0,0 +1,100 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "./index.ts"; + +test("log batches initialize traces from each item's trace ID and normalized timestamp", async () => { + const store = createTelemetryStore(); + const envelope = [ + {}, + [ + [ + { type: "log" }, + { + items: [ + { + id: "one", + trace_id: "logs", + timestamp: 1700000001, + body: "first", + level: "info", + }, + { + id: "two", + trace_id: "other", + timestamp: 1700000002, + body: "other trace", + level: "info", + }, + { + id: "three", + trace_id: "logs", + timestamp: 1700000003, + body: "last", + level: "info", + }, + { + id: "invalid-id", + trace_id: 123, + timestamp: 1700000004, + body: "invalid trace", + level: "info", + }, + { + id: "invalid-time", + trace_id: "bad-time", + timestamp: "invalid", + body: "invalid time", + level: "info", + }, + ], + }, + ], + ], + ]; + const raw = structuredClone(envelope); + await store.ingestEnvelope(envelope); + expect(store.getTrace("logs")).toMatchObject({ + trace_id: "logs", + start_timestamp: 1700000001000, + timestamp: 1700000003000, + spanTree: [], + transactions: [], + }); + expect(store.getTrace("logs")?.spans.size).toBe(0); + expect( + store + .getTraces() + .map(trace => trace.trace_id) + .sort(), + ).toEqual(["logs", "other"]); + expect(store.getLogs({ traceId: "logs" })).toHaveLength(2); + expect(envelope).toEqual(raw); + + await store.ingestEnvelope([ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { + trace_id: "logs", + span_id: "root", + is_segment: true, + name: "GET /later", + start_timestamp: 1700000000, + end_timestamp: 1700000002, + }, + ], + }, + ], + ], + ]); + expect(store.getTrace("logs")).toMatchObject({ + rootTransactionName: "GET /later", + start_timestamp: 1700000000000, + timestamp: 1700000003000, + }); + expect(store.getTrace("logs")?.spans.size).toBe(1); + store.dispose(); +}); diff --git a/packages/spotlight/src/telemetry/reader.spec.ts b/packages/spotlight/src/telemetry/reader.spec.ts index 77a5a80b9..f8b230afb 100644 --- a/packages/spotlight/src/telemetry/reader.spec.ts +++ b/packages/spotlight/src/telemetry/reader.spec.ts @@ -5,8 +5,20 @@ import { createTelemetryReader } from "./reader.ts"; import type { Span, Trace } from "./types.ts"; function fixture() { const spans: Span[] = [ - { span_id: "b", start_timestamp: 10, timestamp: 20, op: "db.query", description: "SELECT 1" }, - { span_id: "a", start_timestamp: 5, timestamp: 20, op: "db.query", description: "SELECT 1" }, + { + span_id: "b", + start_timestamp: 10, + timestamp: 20, + op: "db.query", + description: "SELECT 1", + }, + { + span_id: "a", + start_timestamp: 5, + timestamp: 20, + op: "db.query", + description: "SELECT 1", + }, { span_id: "c", start_timestamp: 25, @@ -32,9 +44,21 @@ function fixture() { revision: 1, envelopes: [], events: [ - { event_id: "b", timestamp: 20, exception: { value: undefined, values: [] } }, - { event_id: "a", timestamp: 20, exception: { value: undefined, values: [] } }, - { event_id: "c", timestamp: 30, exception: { value: undefined, values: [] } }, + { + event_id: "b", + timestamp: 20, + exception: { value: undefined, values: [] }, + }, + { + event_id: "a", + timestamp: 20, + exception: { value: undefined, values: [] }, + }, + { + event_id: "c", + timestamp: 30, + exception: { value: undefined, values: [] }, + }, ], traces: [trace], logs: [], @@ -76,18 +100,33 @@ test("summaries default to compact and preserve trace-less legacy spans", () => const { snapshot } = fixture(); const reader = createTelemetryReader(() => snapshot); expect(reader.getDatabaseQuerySummaries({ traceId: "trace", toMs: 20 })).toEqual([ - { description: "SELECT 1", count: 2, totalDurationMs: 25, averageDurationMs: 12.5 }, + { + description: "SELECT 1", + count: 2, + totalDurationMs: 25, + averageDurationMs: 12.5, + }, ]); expect(reader.getDatabaseQuerySummaries({ includeSpans: true })[0].spans).toHaveLength(2); expect(reader.getResourceSummaries()).toEqual([ - { description: "script.js", count: 1, totalDurationMs: 5, averageDurationMs: 5, averageEncodedSizeBytes: 40 }, + { + description: "script.js", + count: 1, + totalDurationMs: 5, + averageDurationMs: 5, + averageEncodedSizeBytes: 40, + averageSizeBytes: 40, + }, ]); expect(reader.getResourceSummaries({ includeSpans: true })[0].spans).toHaveLength(1); }); test("canonical filters share cached identity until revision changes", () => { let { snapshot } = fixture(); const reader = createTelemetryReader(() => snapshot); - const first = reader.getDatabaseQuerySummaries({ traceId: "trace", fromMs: 10 }); + const first = reader.getDatabaseQuerySummaries({ + traceId: "trace", + fromMs: 10, + }); expect(reader.getDatabaseQuerySummaries({ fromMs: 10, traceId: "trace" })).toBe(first); expect(reader.getEvents()).toBe(reader.getEvents({})); snapshot = { ...snapshot, revision: 2 }; @@ -97,9 +136,20 @@ test("page cursors bind collection, canonical query and revision", () => { let { snapshot } = fixture(); const reader = createTelemetryReader(() => snapshot); const first = reader.readPage("events", { limit: 1, fromMs: 20 }); - expect(first).toMatchObject({ status: "ok", revision: 1, total: 3, items: [{ event_id: "a" }] }); + expect(first).toMatchObject({ + status: "ok", + revision: 1, + total: 3, + items: [{ event_id: "a" }], + }); if (first.status !== "ok") throw new Error("expected page"); - expect(reader.readPage("events", { fromMs: 20, limit: 1, cursor: first.nextCursor })).toMatchObject({ + expect( + reader.readPage("events", { + fromMs: 20, + limit: 1, + cursor: first.nextCursor, + }), + ).toMatchObject({ status: "ok", items: [{ event_id: "b" }], }); @@ -107,17 +157,35 @@ test("page cursors bind collection, canonical query and revision", () => { status: "invalid", code: "invalid_cursor", }); - expect(reader.readPage("events", { fromMs: 21, limit: 1, cursor: first.nextCursor })).toMatchObject({ + expect( + reader.readPage("events", { + fromMs: 21, + limit: 1, + cursor: first.nextCursor, + }), + ).toMatchObject({ status: "invalid", code: "invalid_cursor", }); snapshot = { ...snapshot, revision: 2 }; - expect(reader.readPage("events", { fromMs: 20, limit: 1, cursor: first.nextCursor })).toMatchObject({ + expect( + reader.readPage("events", { + fromMs: 20, + limit: 1, + cursor: first.nextCursor, + }), + ).toMatchObject({ status: "invalid", code: "stale_cursor", }); - expect(reader.readPage("events", { cursor: "bad" })).toMatchObject({ status: "invalid", code: "invalid_cursor" }); - expect(reader.readPage("events", { limit: 1001 })).toMatchObject({ status: "invalid", code: "invalid_query" }); + expect(reader.readPage("events", { cursor: "bad" })).toMatchObject({ + status: "invalid", + code: "invalid_cursor", + }); + expect(reader.readPage("events", { limit: 1001 })).toMatchObject({ + status: "invalid", + code: "invalid_query", + }); expect(reader.readPage("events", { fromMs: 30, toMs: 20 })).toMatchObject({ status: "invalid", code: "invalid_query", @@ -162,7 +230,10 @@ test("web vitals apply limit after selecting measurements", () => { fid: { value: 10, unit: "millisecond" }, }, }; - const reader = createTelemetryReader(() => ({ ...snapshot, events: [...snapshot.events, measured] })); + const reader = createTelemetryReader(() => ({ + ...snapshot, + events: [...snapshot.events, measured], + })); expect(reader.getWebVitals({ limit: 1 }).map(event => event.event_id)).toEqual(["vital"]); }); @@ -176,5 +247,7 @@ test("reader bounds cached queries while retaining recent identities", () => { expect(reader.getEvents({ fromMs: 0 })).toBe(recent); } expect(reader.getEvents({ fromMs: -1 })).not.toBe(oldest); - expect(reader.readPage("events", { limit: 0 })).toMatchObject({ revision: 1 }); + expect(reader.readPage("events", { limit: 0 })).toMatchObject({ + revision: 1, + }); }); diff --git a/packages/spotlight/src/telemetry/selectors/resources.ts b/packages/spotlight/src/telemetry/selectors/resources.ts index 0035dcbbf..6f9ae9b73 100644 --- a/packages/spotlight/src/telemetry/selectors/resources.ts +++ b/packages/spotlight/src/telemetry/selectors/resources.ts @@ -1,15 +1,42 @@ import type { Span } from "../types.ts"; import { type SpanSummary, summarizeSpans } from "./spans.ts"; -export type ResourceSummary = SpanSummary & { averageEncodedSizeBytes: number }; +export type ResourceSummary = SpanSummary & { + averageEncodedSizeBytes: number; + /** Best available captured response size. Unknown sizes are excluded. */ + averageSizeBytes?: number; +}; + +function knownResourceSize(span: Span): number | undefined { + for (const key of [ + "http.response_content_length", + "http.response.body.size", + "http.decoded_response_content_length", + ]) { + const value = span.data?.[key]; + if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value; + } + return undefined; +} export function getResourceSummaries(spans: readonly Span[]): ResourceSummary[] { - return summarizeSpans(spans.filter(span => /resource\.[A-Za-z]+/.test(span.op ?? ""))).map(group => ({ - ...group, - averageEncodedSizeBytes: - group.spans.reduce((sum, span) => { - const size = span.data?.["http.response_content_length"]; - return sum + (typeof size === "number" ? size : 0); - }, 0) / group.count, - })); + return summarizeSpans(spans.filter(span => /resource\.[A-Za-z]+/.test(span.op ?? ""))).map(group => { + const sizes = group.spans.flatMap(span => { + const size = knownResourceSize(span); + return size === undefined ? [] : [size]; + }); + return { + ...group, + averageEncodedSizeBytes: + group.spans.reduce((sum, span) => { + const size = span.data?.["http.response_content_length"]; + return sum + (typeof size === "number" && Number.isFinite(size) && size >= 0 ? size : 0); + }, 0) / group.count, + ...(sizes.length + ? { + averageSizeBytes: sizes.reduce((sum, size) => sum + size, 0) / sizes.length, + } + : {}), + }; + }); } diff --git a/packages/spotlight/src/telemetry/selectors/selectors.test.ts b/packages/spotlight/src/telemetry/selectors/selectors.test.ts index 974debe7f..2e145911b 100644 --- a/packages/spotlight/src/telemetry/selectors/selectors.test.ts +++ b/packages/spotlight/src/telemetry/selectors/selectors.test.ts @@ -35,7 +35,11 @@ function profile(): SentryProcessedProfile { describe("domain summaries", () => { it("groups database spans using normalized millisecond fields and excludes invalid inputs", () => { const first = span(); - const second = span({ span_id: "second", description: "SELECT 1", timestamp: 1030 }); + const second = span({ + span_id: "second", + description: "SELECT 1", + timestamp: 1030, + }); const result = getDatabaseQuerySummaries([ first, span({ op: "http.client" }), @@ -61,9 +65,15 @@ describe("domain summaries", () => { }); it("reports resource size in bytes and retains member spans once", () => { - const first = span({ op: "resource.script", data: { "http.response_content_length": 400 } }); + const first = span({ + op: "resource.script", + data: { "http.response_content_length": 400 }, + }); const second = span({ op: "resource.script", timestamp: 1030 }); - const third = span({ op: "resource.script", data: { "http.response_content_length": "unknown" } }); + const third = span({ + op: "resource.script", + data: { "http.response_content_length": "unknown" }, + }); expect(getResourceSummaries([first, second, third, span()])).toEqual([ { description: "SELECT 1", @@ -71,6 +81,7 @@ describe("domain summaries", () => { totalDurationMs: 50, averageDurationMs: 50 / 3, averageEncodedSizeBytes: 400 / 3, + averageSizeBytes: 400, spans: [first, second, third], }, ]); @@ -82,15 +93,60 @@ describe("domain summaries", () => { ["trace-b", profile()], ]); expect(getProfileFunctionSummaries(profiles)).toEqual([ - { name: "app@main", totalDurationMs: 80, sampleCount: 4, traceIds: new Set(["trace-a", "trace-b"]) }, - { name: "app@query", totalDurationMs: 20, sampleCount: 2, traceIds: new Set(["trace-a", "trace-b"]) }, + { + name: "app@main", + totalDurationMs: 80, + sampleCount: 4, + traceIds: new Set(["trace-a", "trace-b"]), + }, + { + name: "app@query", + totalDurationMs: 20, + sampleCount: 2, + traceIds: new Set(["trace-a", "trace-b"]), + }, ]); expect(getProfileFunctionSummaries(profiles, { traceId: "trace-a" })).toEqual([ - { name: "app@main", totalDurationMs: 40, sampleCount: 2, traceIds: new Set(["trace-a"]) }, - { name: "app@query", totalDurationMs: 10, sampleCount: 1, traceIds: new Set(["trace-a"]) }, + { + name: "app@main", + totalDurationMs: 40, + sampleCount: 2, + traceIds: new Set(["trace-a"]), + }, + { + name: "app@query", + totalDurationMs: 10, + sampleCount: 1, + traceIds: new Set(["trace-a"]), + }, ]); expect(getProfileFunctionSummaries(profiles, { traceId: "missing" })).toEqual([]); expect(getProfileFunctionSummaries(new Map([["empty", { ...profile(), samples: [] }]]))).toEqual([]); expect(profiles.get("trace-a")?.samples).toHaveLength(3); }); }); + +it("averages known resource sizes across captured aliases without treating absent or invalid sizes as zero", () => { + const sizes = [ + { "http.response_content_length": 400 }, + { "http.response.body.size": 200 }, + { "http.decoded_response_content_length": 300 }, + { + "http.response_content_length": "invalid", + "http.response.body.size": 100, + }, + {}, + { "http.response.body.size": -1 }, + { "http.response.body.size": Number.NaN }, + { "http.response.body.size": 0 }, + ]; + const result = getResourceSummaries( + sizes.map((data, index) => span({ span_id: String(index), op: "resource.script", data })), + ); + expect(result[0]).toMatchObject({ + count: 8, + averageEncodedSizeBytes: 50, + averageSizeBytes: 200, + }); + expect(getResourceSummaries([span({ op: "resource.script" })])[0]).not.toHaveProperty("averageSizeBytes"); +}); diff --git a/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts index 5ef33c827..f957bb9a3 100644 --- a/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts +++ b/packages/spotlight/src/telemetry/store/slices/eventsSlice.ts @@ -62,7 +62,7 @@ export const createEventsSlice = } if (isLogEvent(event) && event.items?.length) { - const { logsById, logsByTraceId } = get(); + const { logsById, logsByTraceId, tracesById } = get(); const existingLogIds = new Set(logsById.keys()); const { processedLogs } = processLogItems(event, existingLogIds); @@ -70,19 +70,40 @@ export const createEventsSlice = // Build up the Maps incrementally to avoid stale state capture const newLogsById = new Map(logsById); const newLogsByTraceId = new Map(logsByTraceId); + const newTracesById = new Map(tracesById); for (const logItem of processedLogs) { newLogsById.set(logItem.id, logItem); - if (logItem.trace_id) { + if (typeof logItem.trace_id === "string" && logItem.trace_id.trim()) { const logSet = newLogsByTraceId.get(logItem.trace_id) || new Set(); logSet.add(logItem); newLogsByTraceId.set(logItem.trace_id, logSet); + // A log batch carries trace IDs on its individual records, not + // necessarily on the containing event. Keep log-only traces + // discoverable until transactions or spans arrive later. + if (Number.isFinite(logItem.timestamp)) { + const trace = + newTracesById.get(logItem.trace_id) ?? + initializeTrace({ + ...event, + timestamp: logItem.timestamp, + start_timestamp: undefined, + contexts: { trace: { trace_id: logItem.trace_id } }, + }); + trace.start_timestamp = Math.min(trace.start_timestamp, logItem.timestamp); + trace.timestamp = Math.max(trace.timestamp, logItem.timestamp); + newTracesById.set(logItem.trace_id, trace); + } } } // Set state once with all accumulated changes - set({ logsById: newLogsById, logsByTraceId: newLogsByTraceId }); + set({ + logsById: newLogsById, + logsByTraceId: newLogsByTraceId, + tracesById: newTracesById, + }); if (runtime.generation !== generation) return; } @@ -151,7 +172,10 @@ export const createEventsSlice = if (mergedV2Profile) { const newProfilesByTraceId = new Map(profilesByTraceId); newProfilesByTraceId.set(trace.trace_id, mergedV2Profile); - set({ tracesById: newTracesById, profilesByTraceId: newProfilesByTraceId }); + set({ + tracesById: newTracesById, + profilesByTraceId: newProfilesByTraceId, + }); } else { set({ tracesById: newTracesById }); } diff --git a/packages/spotlight/src/telemetry/topology.spec.ts b/packages/spotlight/src/telemetry/topology.spec.ts new file mode 100644 index 000000000..0ab4449ab --- /dev/null +++ b/packages/spotlight/src/telemetry/topology.spec.ts @@ -0,0 +1,30 @@ +import { expect, test } from "vitest"; +import { createTelemetryStore } from "../sdk/core.ts"; + +test("a self-parented span remains visible as a tree root", async () => { + const telemetry = createTelemetryStore(); + try { + const result = await telemetry.ingestEnvelope([ + {}, + [ + [ + { type: "span" }, + { + version: 2, + items: [ + { trace_id: "trace", span_id: "self", parent_span_id: "self", start_timestamp: 1, end_timestamp: 2 }, + { trace_id: "trace", span_id: "child", parent_span_id: "self", start_timestamp: 1.1, end_timestamp: 1.5 }, + ], + }, + ], + ], + ]); + expect(result.status).toBe("processed"); + const trace = telemetry.getTrace("trace")!; + expect(trace.spans.size).toBe(2); + expect(trace.spanTree.map(span => span.span_id)).toEqual(["self"]); + expect(trace.spanTree[0].children?.map(span => span.span_id)).toEqual(["child"]); + } finally { + telemetry.dispose(); + } +}); diff --git a/packages/spotlight/src/telemetry/utils/traces.ts b/packages/spotlight/src/telemetry/utils/traces.ts index cab6dfd96..6afd5ee94 100644 --- a/packages/spotlight/src/telemetry/utils/traces.ts +++ b/packages/spotlight/src/telemetry/utils/traces.ts @@ -18,7 +18,7 @@ export function groupSpans(spans: Map): Span[] { if (parent) { parent.children ??= []; parent.children.push(span); - } else if (span.parent_span_id) { + } else if (span.parent_span_id && span.parent_span_id !== span.span_id) { const parentParent = sortedSpans.find(s => !s.parent_span_id); if (!parentParent) { log(`Root span (${span.parent_span_id}) for span (${span.span_id}). Creating orphan.`); @@ -54,7 +54,7 @@ export function groupSpans(spans: Map): Span[] { } function getParentOfSpan(span: Span, idLookup: Map, allSpans: Span[]): Span | undefined { - if (!span.parent_span_id) { + if (!span.parent_span_id || span.parent_span_id === span.span_id) { return undefined; }