feat(sdk): Share portable telemetry processing across consumers

.github/workflows/build.yml

@@ -113,6 +113,9 @@ jobs:
           FOSSILIZE_PLATFORMS: linux-x64,linux-arm64,win-x64,darwin-x64,darwin-arm64
           FOSSILIZE_SIGN: ${{ github.event_name == 'push' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) && 'y' || 'n' }}
         run: pnpm build
+
+      - name: Check built SDK types and run examples
+        run: pnpm test:sdk-types
 
       - name: Checking npx
         run: npx ./packages/spotlight --help

README.md

@@ -36,6 +36,12 @@ Spotlight offers two main ways to work with your development telemetry:
 
 Spotlight uses Sentry SDKs to capture telemetry data and sends it to the Sidecar, a local proxy server running on your machine. The Sidecar processes these events and forwards them to the UI, where you can view everything in real-time via the web interface or stream output to your terminal. Spotlight works standalone or alongside your existing Sentry setup without interference.
 
+## Embed the telemetry SDK
+
+Use `createTelemetryStore()` from `@spotlightjs/spotlight/sdk/core` to process parsed Sentry envelopes in portable runtimes, or `createTelemetryPipeline()` from `@spotlightjs/spotlight/sdk/node` for serialized and compressed Node payloads. Both share the same processing, receipts, readers, and retention controls.
+
+See the [SDK reference](packages/spotlight/README.md) and [executable examples](packages/spotlight/examples/sdk) for integration, retries, cancellation, paging, JSON exports, and React subscriptions.
+
 ## Links
 
 - [![Documentation](https://img.shields.io/badge/documentation-spotlight-indigo.svg)](https://spotlightjs.com/about/)

package.json

@@ -6,6 +6,7 @@
     "dev:website": "turbo dev --filter=@spotlightjs/website",
     "dev:electron": "pnpm run --filter=@spotlightjs/spotlight dev:electron",
     "build": "turbo build",
+    "test:sdk-types": "pnpm --filter @spotlightjs/spotlight test:sdk-types",
     "preview": "turbo preview",
     "lint": "biome check .",
     "lint:fix": "biome check --write .",

packages/spotlight/README.md

@@ -1,5 +1,168 @@
 # `@spotlightjs/spotlight`
 
-This package combines the UI from `@spotlightjs/overlay` and the sidecar from `@spotlightjs/sidecar`.
+Spotlight provides a development telemetry UI, a local sidecar, and a desktop app.
+Its SDK shares normalization, trace assembly, and insight calculations across
+server and frontend consumers. Each instance owns its state; feed server and
+frontend instances the same envelope stream and match retention settings when
+comparing their retained histories.
 
-This is the package to use for Spotlight.
+## Choose an entry point
+
+| Entry | Factory | Input |
+| --- | --- | --- |
+| `@spotlightjs/spotlight/sdk/core` | `createTelemetryStore()` | Parsed Sentry envelopes in portable runtimes |
+| `@spotlightjs/spotlight/sdk/node` | `createTelemetryPipeline()` | Adds serialized bodies, decompression, and decoded payload history |
+| `@spotlightjs/spotlight/sdk` | Existing buffer helpers | Compatibility for existing transport consumers |
+
+## Ingest and read telemetry
+
+Use `@spotlightjs/spotlight/sdk/core` for parsed Sentry envelopes in a browser, worker, or Node application. Use `@spotlightjs/spotlight/sdk/node` for serialized or compressed payloads on Node 20+. Both expose the same processing and read APIs; creating an instance does not start a server. The portable entry has no Node or React dependency. The existing raw-buffer helpers at `/sdk` remain available.
+
+```ts
+import { createTelemetryStore, serializeSnapshot } from "@spotlightjs/spotlight/sdk/core";
+
+const telemetry = createTelemetryStore();
+const receipt = await telemetry.ingestEnvelope(parsedEnvelope, {
+  deliveryId: "transport-request-123", // Reuse for retries of this delivery.
+  signal: abortController.signal,
+});
+if (receipt.status === "processed") {
+  console.log(receipt.envelopeId, receipt.counts, receipt.retained);
+} else if (receipt.status === "ignored") {
+  console.log(receipt.reason); // For example: duplicate, aborted, queue_full.
+} else {
+  console.log(receipt.diagnostics); // Literal codes, severity, optional itemIndex/spanIndex.
+}
+await telemetry.drain();
+const json = JSON.stringify(serializeSnapshot(telemetry.getSnapshot()));
+telemetry.dispose();
+```
+
+The Node adapter accepts `string`, `Uint8Array`, and `Buffer` bodies. `ingestPayload({ body, contentType?, encoding?, userAgent?, deliveryId?, signal? })` decodes `application/x-sentry-envelope` (the default content type) and supports identity, gzip, deflate, and Brotli encoding. Use `getDecodedPayload(envelopeId)` for a copied decompressed payload when it remains in the independently bounded history. Parsed envelopes still use `ingestEnvelope`; the payload methods are `ingestPayload` and `ingestPayloads`.
+
+```ts
+import { createTelemetryPipeline } from "@spotlightjs/spotlight/sdk/node";
+
+const pipeline = createTelemetryPipeline({ decodedHistory: { maxEntries: 100 } });
+const receipt = await pipeline.ingestPayload({ body: envelopeBody, deliveryId: "request-1" });
+if (receipt.status === "processed") {
+  const payload = pipeline.getDecodedPayload(receipt.envelopeId); // Defensive decoded-byte copy.
+  const queries = pipeline.getDatabaseQuerySummaries({ traceId }); // Compact by default.
+}
+await pipeline.drain();
+pipeline.dispose();
+```
+
+### Receipts, retries, and lifecycle
+
+Every input returns an immutable receipt with `status`, `diagnostics`, `revision`, `retained`, and `counts`:
+
+| Field | Meaning |
+| --- | --- |
+| `status: "processed"` | A supported envelope was processed. This does not imply new telemetry or continued retention. |
+| `status: "ignored"` | `reason` explains the outcome: `duplicate`, `unsupported_items`, `unsupported_content_type`, `reset`, `disposed`, `aborted`, or `queue_full`. |
+| `status: "invalid"` | Diagnostic codes explain invalid data or exceeded payload limits. |
+| `retained` | Whether this envelope is in processed retention when the receipt is produced. An unsupported envelope can be retained; an immediately evicted processed envelope is not. This says nothing about Node decoded history. |
+| `revision` | Published snapshot revision at the outcome. A retry that changes no state need not advance it. |
+| `counts.events`, `counts.spans`, `counts.logs` | Newly added event IDs, trace/span pairs, and log IDs during processing, before retention eviction. These overlap: standalone spans can also become events. They are not retained collection sizes. |
+| `counts.skippedItems` | Unsupported wire items plus malformed entries skipped inside supported span containers. |
+| `counts.duplicateEvents` | Candidate event observations that did not add a new event ID. Delivery-level duplicate receipts have zero counts. |
+
+Branch on diagnostic `code` and `severity`, or ignored `reason`, rather than message text. Diagnostics can accompany successful partial processing. `itemIndex` and `spanIndex`, when present, are zero-based locations.
+
+Delivery IDs deduplicate only while the corresponding envelope is retained. Eviction and `reset()` permit that delivery again. There is no content hashing: separately allocated envelopes without IDs are separate observations. Reusing the same parsed object or its explicit Spotlight envelope ID preserves its delivery identity within the retained window. Event-ID deduplication is separate from delivery deduplication.
+
+`ingestEnvelopes(iterable, { signal? })` returns receipts in input order and preserves valid neighbors of invalid inputs. It is not a transaction. `ingestPayloads(inputs)` does the same for Node inputs; each input carries its own signal and delivery ID. A batch occupies one queued operation, including while active. Items are copied as they are reached rather than copying the entire batch up front: keep queued batch items unchanged until their receipts resolve. Individual `ingestEnvelope` and `ingestPayload` calls copy their input before the first await. Cancellation returns `ignored/aborted`. `drain()` waits for operations already submitted at the call, excluding future work. `getStats()` reports pending operations, retained envelope count/bytes, revision, and disposal. `subscribe((snapshot, change) => ...)` returns an unsubscribe function; `change` identifies ingest, eviction, reset, or disposal and affected envelope IDs. Release timers and listeners with `dispose()`.
+
+An optional `enrichStacktrace(event, { signal })` hook receives a copied event. Observe its signal to cancel external work. A timeout reports a warning and keeps telemetry; abort, reset, disposal, or timeout prevent late hook writes from entering published state.
+
+### Read APIs and JSON
+
+Accept a `TelemetryReader` from `/sdk/core` in consumers that only query telemetry. Both factories implement it. `createTelemetryReader(getSnapshot)` also constructs a reader over an existing snapshot source. Singular getters such as `getTrace(id)` and `getEvent(id)` return `undefined` when absent; collection getters return readonly arrays. All processed timestamps and durations use milliseconds; time bounds are inclusive Unix milliseconds.
+
+```ts
+const queries = reader.getDatabaseQuerySummaries({ traceId, fromMs, toMs, limit: 10 });
+const detailed = reader.getDatabaseQuerySummaries({ traceId, includeSpans: true });
+const first = reader.readPage("events", { traceId, limit: 100, order: "desc" });
+if (first.status === "ok" && first.nextCursor) {
+  const next = reader.readPage("events", {
+    traceId, limit: 100, order: "desc", cursor: first.nextCursor,
+  });
+  // Handle next.status === "invalid" before using next.items.
+}
+```
+
+Queries and resource summaries omit full span lists by default; opt in with `includeSpans: true`. Getters accept `fromMs`, `toMs`, `order` (`asc` by default), and a positive `limit`; getters have no implicit limit. Chronological ties use stable IDs. Summary rows sort by description and apply time bounds to their input spans. Invalid getter options throw `RangeError`.
+
+`readPage` supports `events`, `traces`, `logs`, and `spans`, defaults to 100 rows, and caps a page at 1000. It returns `status: "ok"`, `revision`, `items`, `total`, and optional `nextCursor`, or `status: "invalid"` with `invalid_query`, `invalid_cursor`, or `stale_cursor`. Cursors bind the collection, filters, order, limit, and snapshot revision. Keep the query unchanged between pages and restart after a stale cursor. `spanId` filtering on pages is supported for events only. Treat cursors as opaque.
+
+Snapshots preserve readonly maps and sets for in-process use. For JSON, use `serializeSnapshot(snapshot)` or `serializeTrace(trace)`: the schema version is `1`, maps become entry arrays, sets become value arrays, and span trees use IDs and `childSpanIds` instead of repeated object graphs. Unsupported non-wire values or unrelated cycles throw `TypeError`. These DTOs are export formats, not inputs to `ingestEnvelope`. Equivalent reads within the current revision reuse cached results; older snapshots remain stable after subsequent ingestion.
+
+Use `serializeSummaries(reader.getProfileFunctionSummaries())` or pass compact query/resource summaries to export `{ schemaVersion: 1, summaries }`. Profile `traceIds` sets become string arrays. Summaries requested with `includeSpans: true` are rejected at both the type and runtime boundaries; export the complete span graph with `serializeTrace` instead.
+
+### Limits and supported formats
+
+| Option | Default |
+| --- | --- |
+| `maxPending` (ingestion operations, including active work) | 64 |
+| `maxEnvelopeBytes` (serialized parsed envelope) | 32 MiB |
+| `enrichmentTimeoutMs` | 5000 ms |
+| `retention.maxEnvelopes` / `retention.maxBytes` | 500 / 64 MiB |
+| `retention.maxAgeMs` | No age limit; when configured, expires automatically since receipt |
+| Node `maxPayloadBytes` / `maxDecodedBytes` | 8 MiB / 32 MiB |
+| Node `decodedHistory.maxEntries` / `decodedHistory.maxBytes` | 500 / 32 MiB |
+
+`setRetention({ maxEnvelopes?, maxBytes?, maxAgeMs? })` updates only supplied retention limits and resolves after queued eviction. Existing subscriptions and Node decoded history remain available. Age limits cannot be cleared; disposed instances ignore updates. Node updates follow earlier payload/parsed work, and `drain()` includes retention updates submitted before the drain.
+
+Limits must be positive safe integers; Node `decodedHistory.maxEntries` and `decodedHistory.maxBytes` also allow zero to disable decoded history. Byte limits use bytes, with 1 MiB = 1024 × 1024 bytes. The Node decoder bounds decompression output. Bound transport collection separately before passing a complete body to the SDK, as shown in the HTTP example. Retention evicts whole envelopes and the telemetry derived from them; the byte budget measures the serialized retained envelopes, not total JavaScript heap size.
+
+`SDK_CAPABILITIES` exposes readonly machine-readable metadata for legacy and v2 spans, profile v1/v2, supported item types, and timestamp units. Supported item types include event/error, transaction, span, log, profile/profile_chunk, feedback, and attachment; individual wire versions can still be unsupported. Portable runtimes need standard JavaScript APIs including `AbortController`, `TextEncoder`, timers, maps, and sets. React is required only by your React consumer; compressed payload decoding is provided by the Node entry.
+
+The [SDK examples](examples/sdk) cover a bounded HTTP server, a frontend transport adapter, a reader-only consumer, React subscription, and an executable lifecycle walkthrough with receipts, retry, partial batches, cancellation, drain, JSON, and paging. After building the Node and UI outputs, run `pnpm test:sdk-types` to typecheck the published declarations and every example, then execute the lifecycle walkthrough. CI runs this check after the package build; it does not add a whole-project TypeScript gate.
+
+### Getter reference
+
+| Read | Result |
+| --- | --- |
+| `getTrace(traceId)`, `getTraces(filter?)` | Assembled traces |
+| `getEvent(eventId)`, `getEvents(filter?)`, `getErrors(filter?)` | Normalized events and captured exceptions |
+| `getSpans(filter?)` | Normalized spans |
+| `getLog(logId)`, `getLogs(filter?)` | Logs and trace correlation |
+| `getDatabaseQuerySummaries(filter?)` | Database spans grouped by trimmed description |
+| `getResourceSummaries(filter?)` | Resource timings and encoded sizes |
+| `getProfile(traceId)`, `getProfileFunctionSummaries(filter?)` | Associated profiles and inclusive frame durations/sample counts |
+| `getAITraces(filter?)` | AI interactions, tokens, prompts, responses, and tools |
+| `getWebVitals(filter?)` | Scored page-load events |
+| `getEnvelope(envelopeId)`, `getEnvelopes()` | Retained wire envelopes |
+| `getSdks()` | Observed SDK identities |
+
+Event reads accept `traceId` and `spanId`; trace/log/span/profile/summary reads
+accept `traceId`; query summaries also accept `description`. The time/order/limit
+filters apply to filtered collection reads, not `getEnvelopes()` or `getSdks()`.
+Summary fields name units explicitly, including `totalDurationMs`,
+`averageDurationMs`, and `averageEncodedSizeBytes`. Resource summaries also expose optional `averageSizeBytes`, averaging known finite, nonnegative response sizes using encoded length, body size, and decoded length fallbacks. Unknown sizes are excluded; `averageEncodedSizeBytes` remains the encoded-specific field.
+
+Stored envelopes retain wire
+timestamp units. Caller input is copied; read results are readonly.
+
+`reset()` clears captured data and decoded history, keeps subscriptions, and
+invalidates in-flight/queued work. `prune()` explicitly enforces retention.
+`dispose()` closes the instance; later ingestion returns `ignored/disposed`.
+Eviction rebuilds derived state from retained envelopes, with cost proportional
+to retained telemetry. Choose smaller limits for large profiles or heavy streams.
+
+### Examples and compatibility
+
+- [HTTP server](examples/sdk/server.ts): bounded `POST /envelope` input and query summaries at `GET /queries?traceId=...`; call `server.listen(...)` to start it.
+- [Frontend adapter](examples/sdk/frontend.ts): one ingester per telemetry view; dispose it when the view disconnects.
+- [Reader-only consumer](examples/sdk/reader.ts): filtered queries, JSON serialization, and bounded page traversal.
+- [React query list](examples/sdk/react.tsx): `useSyncExternalStore` using a reader and subscription, with no ingestion methods exposed to the view.
+- [Executable lifecycle walkthrough](examples/sdk/lifecycle.ts): checked and run by `pnpm test:sdk-types` after the package build.
+
+Snapshots retain the same identity until a committed update, suitable for
+`useSyncExternalStore`. Subscribers see coherent snapshots; a failing subscriber
+or diagnostic observer cannot break ingestion. Released `createSpotlightBuffer`,
+`pushToSpotlightBuffer`, and `decompressBody` remain available from `/sdk` with
+their existing behavior. Raw buffering alone does not invoke processing.
+
+Pruning uses one coalesced control operation outside ingestion admission; it remains part of the drain barrier.

packages/spotlight/_fixtures/continuous_profiling/profile_chunk_1.txt

@@ -1,3 +1,3 @@
 {"event_id":"3e11a5c9831f4e49939c0a81944ea2cb","sent_at":"2024-08-27T15:00:11.503779Z","sdk":{"name":"sentry.cocoa","version":"8.36.0"}}
-{"type":"profile_chunk","length":1671}
+{"type":"profile_chunk","length":1643}
 {"type":"profile_chunk","version":"2","profiler_id":"71bba98d90b545c39f2ae73f702d7ef4","chunk_id":"3e11a5c9831f4e49939c0a81944ea2cb","platform":"cocoa","release":"io.sentry.sample.iOS-Swift@8.36.0+1","environment":"simulator","client_sdk":{"name":"sentry.cocoa","version":"8.36.0"},"debug_meta":{"images":[{"debug_id":"5819FF25-01CB-3D32-B84F-0634B37D3BBC","image_addr":"0x00000001023a8000","type":"macho","image_size":16384,"code_file":"/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS17.2.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libLogRedirect.dylib"}]},"profile":{"samples":[{"timestamp":1724770811.503,"stack_id":0,"thread_id":"259"},{"timestamp":1724770811.513,"stack_id":1,"thread_id":"259"},{"timestamp":1724770811.523,"stack_id":2,"thread_id":"259"},{"timestamp":1724770811.533,"stack_id":3,"thread_id":"259"},{"timestamp":1724770811.543,"stack_id":2,"thread_id":"259"},{"timestamp":1724770811.553,"stack_id":1,"thread_id":"259"},{"timestamp":1724770811.563,"stack_id":0,"thread_id":"259"}],"stacks":[[0],[1,0],[2,1,0],[3,2,1,0]],"frames":[{"instruction_addr":"0x000000010232d144","function":"_main","filename":"main.m","lineno":14,"in_app":true},{"instruction_addr":"0x000000010232d200","function":"UIApplicationMain","module":"UIKit","in_app":false},{"instruction_addr":"0x000000010232d300","function":"-[AppDelegate application:didFinishLaunchingWithOptions:]","filename":"AppDelegate.swift","lineno":15,"in_app":true},{"instruction_addr":"0x000000010232d400","function":"-[AppDelegate setupSentry]","filename":"AppDelegate.swift","lineno":25,"in_app":true}],"thread_metadata":{"259":{"name":"main","priority":31}}}}

packages/spotlight/examples/sdk/frontend.ts

@@ -0,0 +1,12 @@
+import { type IngestOptions, createTelemetryStore } from "@spotlightjs/spotlight/sdk/core";
+
+/** Keep one instance for the lifetime of the telemetry view. */
+export function createFrontendTelemetry() {
+  const telemetry = createTelemetryStore({ retention: { maxEnvelopes: 500 } });
+  return {
+    telemetry,
+    // Pass the parsed envelopes received over SSE, WebSocket, or your own transport.
+    receiveEnvelope: (envelope: unknown, options?: IngestOptions) => telemetry.ingestEnvelope(envelope, options),
+    disconnect: telemetry.dispose,
+  };
+}