From 49bd826ccaff76547457f208ef1f7a630bd69e03 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 16 Oct 2026 00:36:15 +0200 Subject: [PATCH 1/4] feat(traces): show span events in the span panel and on the timeline Read the OTel events recorded on spans (including `exception`) from the traces table. The span detail panel gets an Events section with offsets from span start, attributes, and exception type, message and stack trace. The trace timeline marks where each event happened along its span's bar, with exceptions highlighted. Two queries back this: spanEventsQuery loads one span's events, keyed by the span's sorting-key columns and exact start second, and spanEventsByTraceQuery loads event names and times for the whole trace for the markers. spanDetailQuery now also returns the stored SpanName, since the display name is rewritten for HTTP spans. --- .../src/routes/internal/query-engine.http.ts | 50 ++++++++-- apps/api/src/routes/queries.ts | 45 +++++++++ apps/web/src/api/warehouse/traces.ts | 51 ++++++++++ .../components/traces/span-detail-panel.tsx | 2 + .../components/traces/span-events-section.tsx | 96 +++++++++++++++++++ .../services/atoms/warehouse-query-atoms.ts | 5 + apps/web/src/lib/traces/span-events.test.ts | 50 ++++++++++ apps/web/src/lib/traces/span-events.ts | 41 ++++++++ apps/web/src/routes/traces/$traceId.tsx | 16 +++- packages/domain/src/http/query-engine.ts | 43 +++++++++ .../src/__sql_baseline__/catalog.sql | 38 +++++++- .../query-engine/src/benchmark/builders.ts | 23 +++++ packages/query-engine/src/ch/ch.test.ts | 41 ++++++++ packages/query-engine/src/ch/index.ts | 5 + .../query-engine/src/ch/queries/errors.ts | 87 ++++++++++++++++- packages/query-engine/src/index.ts | 1 + .../src/observability/span-detail.test.ts | 1 + packages/query-engine/src/span-events.test.ts | 50 ++++++++++ packages/query-engine/src/span-events.ts | 77 +++++++++++++++ .../components/traces/trace-timeline-row.tsx | 63 +++++++++++- .../src/components/traces/trace-timeline.tsx | 26 ++--- .../components/traces/trace-view-context.tsx | 4 + .../src/components/traces/trace-view-tabs.tsx | 5 + .../lib/__tests__/span-event-markers.test.ts | 34 +++++++ packages/ui/src/lib/span-event-markers.ts | 35 +++++++ 25 files changed, 857 insertions(+), 32 deletions(-) create mode 100644 apps/web/src/components/traces/span-events-section.tsx create mode 100644 apps/web/src/lib/traces/span-events.test.ts create mode 100644 apps/web/src/lib/traces/span-events.ts create mode 100644 packages/query-engine/src/span-events.test.ts create mode 100644 packages/query-engine/src/span-events.ts create mode 100644 packages/ui/src/lib/__tests__/span-event-markers.test.ts create mode 100644 packages/ui/src/lib/span-event-markers.ts diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 7a71bf8860..1cfe4face3 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -7,6 +7,7 @@ import { type RawSqlValidationError, SpanHierarchyResponse, SpanDetailResponse, + TraceSpanEventsResponse, ErrorsByTypeResponse, ErrorsSparkResponse, ErrorsSummaryResponse, @@ -114,6 +115,7 @@ import { computeBucketSecondsForRange, formatWarehouseDateTime, QueryEngineExecuteBatchResponse, + parseSpanEvents, parseWarehouseDateTime, } from "@maple/query-engine" @@ -517,14 +519,48 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query Effect.gen(function* () { const tenant = yield* CurrentTenant.Context const row = yield* runQueryFirst(Queries.spanDetail, tenant, payload) + if (!row) return new SpanDetailResponse({ data: null }) + // `trace_detail_spans` has no Events columns; read them off `traces` + // by the sorting key the detail row just resolved. + const startMs = DateTime.toEpochMillis(row.startTime) + const events = yield* runQueryFirst(Queries.spanEvents, tenant, { + traceId: row.traceId, + spanId: row.spanId, + serviceName: row.serviceName, + spanName: row.rawSpanName, + startTime: formatWarehouseDateTime(startMs), + endTime: formatWarehouseDateTime(startMs + 1000), + }).pipe( + Effect.map((raw) => (raw ? parseSpanEvents(raw) : [])), + // Events are supplementary; a failed lookup must not hide the attributes. + Effect.tapError((error) => Effect.logWarning("span events lookup failed", error)), + Effect.orElseSucceed(() => []), + ) return new SpanDetailResponse({ - data: row - ? { - ...row, - traceId: decodeTraceId(row.traceId), - spanId: decodeSpanId(row.spanId), - } - : null, + data: { + traceId: decodeTraceId(row.traceId), + spanId: decodeSpanId(row.spanId), + spanAttributes: row.spanAttributes, + resourceAttributes: row.resourceAttributes, + events, + }, + }) + }), + ) + .handle("traceSpanEvents", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.traceSpanEvents, tenant, payload) + return new TraceSpanEventsResponse({ + data: rows.flatMap((row) => { + const events = parseSpanEvents(row).map(({ name, offsetMs }) => ({ + name, + offsetMs, + })) + return events.length === 0 + ? [] + : [{ spanId: decodeSpanId(row.spanId), events }] + }), }) }), ) diff --git a/apps/api/src/routes/queries.ts b/apps/api/src/routes/queries.ts index 5eb9100332..607b4821f7 100644 --- a/apps/api/src/routes/queries.ts +++ b/apps/api/src/routes/queries.ts @@ -24,6 +24,7 @@ import type { RailwayInfraServiceTimeseriesRequest, ServiceCloudflareStatsRequest, SpanDetailRequest, + TraceSpanEventsRequest, WorkloadInfraTimeseriesRequest, } from "@maple/domain/http" import { @@ -673,6 +674,50 @@ export const Queries = { }, }), + /** + * Events of a span `spanDetail` already resolved. Keyed by the span's exact + * start second, so the row it reads is written and immutable. + */ + spanEvents: defineQuery({ + id: "spanEvents", + profile: "discovery", + cache: 600, + compile: ( + payload: { + readonly traceId: string + readonly spanId: string + readonly serviceName: string + readonly spanName: string + readonly startTime: string + readonly endTime: string + }, + orgId: OrgId, + ) => + CH.compile( + CH.spanEventsQuery({ + traceId: payload.traceId, + spanId: payload.spanId, + serviceName: payload.serviceName, + spanName: payload.spanName, + }), + { orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), + }), + + /** Event times and names for every span of a trace, for the timeline markers. */ + traceSpanEvents: defineQuery({ + id: "traceSpanEvents", + profile: "list", + cache: (payload: TraceSpanEventsRequest, nowMs: number) => + traceCacheTtlSeconds(payload.endTime, nowMs), + compile: (payload: TraceSpanEventsRequest, orgId: OrgId) => + CH.compile(CH.spanEventsByTraceQuery({ traceId: payload.traceId }), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + }), + }), + podInfraTimeseries: defineQuery({ id: "podInfraTimeseries", profile: "aggregation", diff --git a/apps/web/src/api/warehouse/traces.ts b/apps/web/src/api/warehouse/traces.ts index 1afd469d76..e6fab1714c 100644 --- a/apps/web/src/api/warehouse/traces.ts +++ b/apps/web/src/api/warehouse/traces.ts @@ -15,7 +15,10 @@ import { SpanDetailRequest, SpanHierarchyRequest, SpanName, + TraceSpanEventsRequest, } from "@maple/domain/http" +import type { SpanEvent } from "@maple/query-engine" +import type { SpanEventsBySpanId } from "@maple/ui/lib/span-event-markers" import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" import { computeTraceTimeWindow } from "@/lib/trace-time-window" import { @@ -578,6 +581,7 @@ export type GetSpanDetailInput = Schema.Schema.Type resourceAttributes: Record + events: ReadonlyArray } export function getSpanDetail({ data }: { data: GetSpanDetailInput }) { @@ -612,6 +616,7 @@ const getSpanDetailEffect = Effect.fn("QueryEngine.getSpanDetail")(function* ({ return { spanAttributes: parseAttributes(result.data?.spanAttributes), resourceAttributes: parseAttributes(result.data?.resourceAttributes), + events: result.data?.events ?? [], } satisfies SpanDetailResult }) @@ -757,6 +762,52 @@ const getTracesFacetsEffect = Effect.fn("QueryEngine.getTracesFacets")(function* } }) +// Trace span events — event times and names for every span, drawn as markers on +// the timeline bars. A separate request so the hierarchy never waits on `traces`. + +/** Slack around the trace's own span window for sub-second rounding and late events. */ +const TRACE_EVENTS_SLACK_MS = 1_000 + +const GetTraceSpanEventsInputSchema = Schema.Struct({ + traceId: TraceId, + /** Earliest span start (`SpanHierarchyResponse.traceStartTime`). */ + traceStartTime: Schema.String, + totalDurationMs: Schema.Number, +}) + +export type GetTraceSpanEventsInput = Schema.Schema.Type + +export function getTraceSpanEvents({ data }: { data: GetTraceSpanEventsInput }) { + return getTraceSpanEventsEffect({ data }) +} + +const getTraceSpanEventsEffect = Effect.fn("QueryEngine.getTraceSpanEvents")(function* ({ + data, +}: { + data: GetTraceSpanEventsInput +}) { + const input = yield* decodeInput(GetTraceSpanEventsInputSchema, data, "getTraceSpanEvents") + yield* Effect.annotateCurrentSpan("traceId", input.traceId) + + const startMs = Date.parse(input.traceStartTime) + if (Number.isNaN(startMs)) return new Map() satisfies SpanEventsBySpanId + + const result = yield* runWarehouseQuery("traceSpanEvents", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.traceSpanEvents({ + payload: new TraceSpanEventsRequest({ + traceId: input.traceId, + startTime: formatWarehouseDateTime(startMs - TRACE_EVENTS_SLACK_MS), + endTime: formatWarehouseDateTime(startMs + input.totalDurationMs + TRACE_EVENTS_SLACK_MS), + }), + }) + }), + ) + + return new Map(result.data.map((row) => [row.spanId, row.events])) satisfies SpanEventsBySpanId +}) + // Single facet list for dashboard variables: compiles only the requested UNION // branch server-side and skips the duration-stats companion query, instead of // the full multi-facet scan `getTracesFacets` runs for the traces sidebar. diff --git a/apps/web/src/components/traces/span-detail-panel.tsx b/apps/web/src/components/traces/span-detail-panel.tsx index bac6570120..3ca798882d 100644 --- a/apps/web/src/components/traces/span-detail-panel.tsx +++ b/apps/web/src/components/traces/span-detail-panel.tsx @@ -35,6 +35,7 @@ import { ResourceAttributesSection, } from "@/components/attributes" import { TraceAttributeFilterProvider } from "./trace-attribute-filter-provider" +import { SpanEventsSection } from "./span-events-section" import { useTimezonePreference } from "@/hooks/use-timezone-preference" import { formatTimestampInTimezone } from "@/lib/timezone-format" import { HttpSpanLabel } from "@maple/ui/components/traces/http-span-label" @@ -473,6 +474,7 @@ export function SpanDetailPanel({ )) .onSuccess((detail) => ( <> + > }) { + const entries = Object.entries(attributes) + if (entries.length === 0) return null + return ( + + {entries.map(([key, value]) => ( + + {value} + + ))} + + ) +} + +function SpanEventRow({ event }: { event: SpanEvent }) { + const described = describeSpanEvent(event) + const offset = ( + + {formatEventOffset(event.offsetMs)} + + ) + + if (described.kind === "event") { + return ( +
  • +
    + {offset} + {event.name || "(unnamed)"} +
    + +
    • + ) + } + + return ( +
  • +
    + {offset} + + + {described.type ?? "exception"} + +
    + {described.message && ( +

    + {described.message} +

    + )} + {described.stacktrace && ( + +
    + + + Stack trace + + +
    + + 
    +							{described.stacktrace}
+
    +     +     + )} + +
    • + ) +} + +/** The span's OTel events in time order, with `exception` events expanded. */ +export function SpanEventsSection({ events }: { events: ReadonlyArray }) { + if (events.length === 0) return null + return ( +
    +

    Events ({events.length})

    +
      + {events.map((event, index) => ( + + ))} +
    +
    + ) +} diff --git a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts index 92ee933556..c65a96c494 100644 --- a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts +++ b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts @@ -100,6 +100,7 @@ import { getSpanAttributeValues, getSpanDetail, getSpanHierarchy, + getTraceSpanEvents, getTracesFacetValues, getTracesFacets, listTraces, @@ -501,6 +502,10 @@ export const getReplaysForTraceResultAtom = makeQueryAtomFamily(getReplaysForTra staleTime: 60_000, }) +export const getTraceSpanEventsResultAtom = makeQueryAtomFamily(getTraceSpanEvents, { + staleTime: 60_000, +}) + export const getSpanDetailResultAtom = makeQueryAtomFamily(getSpanDetail, { staleTime: 60_000, }) diff --git a/apps/web/src/lib/traces/span-events.test.ts b/apps/web/src/lib/traces/span-events.test.ts new file mode 100644 index 0000000000..1e27119f2d --- /dev/null +++ b/apps/web/src/lib/traces/span-events.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest" +import { describeSpanEvent, formatEventOffset } from "./span-events" + +const event = (name: string, attributes: Record) => ({ + name, + timestamp: "2026-03-20T08:00:00.000Z", + offsetMs: 0, + attributes, +}) + +describe("describeSpanEvent", () => { + it("pulls the semconv fields out of an exception event", () => { + const described = describeSpanEvent( + event("exception", { + "exception.type": "TypeError", + "exception.message": "x is undefined", + "exception.stacktrace": "TypeError: x is undefined\n at f (a.js:1:1)", + "exception.escaped": "true", + }), + ) + expect(described).toEqual({ + kind: "exception", + type: "TypeError", + message: "x is undefined", + stacktrace: "TypeError: x is undefined\n at f (a.js:1:1)", + attributes: { "exception.escaped": "true" }, + }) + }) + + it("treats blank exception fields as absent", () => { + const described = describeSpanEvent(event("exception", { "exception.message": "" })) + expect(described).toMatchObject({ kind: "exception", message: undefined, stacktrace: undefined }) + }) + + it("passes other events through with all attributes", () => { + expect(describeSpanEvent(event("cache.miss", { "exception.type": "kept" }))).toEqual({ + kind: "event", + attributes: { "exception.type": "kept" }, + }) + }) +}) + +describe("formatEventOffset", () => { + it("signs the offset and keeps microsecond precision", () => { + expect(formatEventOffset(12.4)).toBe("+12.400ms") + expect(formatEventOffset(0.25)).toBe("+250μs") + expect(formatEventOffset(-1.5)).toBe("-1.500ms") + expect(formatEventOffset(2500)).toBe("+2.500s") + }) +}) diff --git a/apps/web/src/lib/traces/span-events.ts b/apps/web/src/lib/traces/span-events.ts new file mode 100644 index 0000000000..c01de06452 --- /dev/null +++ b/apps/web/src/lib/traces/span-events.ts @@ -0,0 +1,41 @@ +import type { SpanEvent } from "@maple/query-engine" +import { formatDurationAtStep } from "@maple/ui/lib/format" + +/** OTel semconv attribute keys on an `exception` span event. */ +const EXCEPTION_KEYS = ["exception.type", "exception.message", "exception.stacktrace"] as const + +export type DescribedSpanEvent = + | { + readonly kind: "exception" + readonly type: string | undefined + readonly message: string | undefined + readonly stacktrace: string | undefined + /** Attributes beyond the three exception keys, e.g. `exception.escaped`. */ + readonly attributes: Readonly> + } + | { readonly kind: "event"; readonly attributes: Readonly> } + +const nonEmpty = (value: string | undefined) => (value ? value : undefined) + +/** Splits an `exception` event's semconv fields from the rest of its attributes. */ +export function describeSpanEvent(event: SpanEvent): DescribedSpanEvent { + if (event.name !== "exception") return { kind: "event", attributes: event.attributes } + const attributes = Object.fromEntries( + Object.entries(event.attributes).filter( + ([key]) => !(EXCEPTION_KEYS as readonly string[]).includes(key), + ), + ) + return { + kind: "exception", + type: nonEmpty(event.attributes["exception.type"]), + message: nonEmpty(event.attributes["exception.message"]), + stacktrace: nonEmpty(event.attributes["exception.stacktrace"]), + attributes, + } +} + +/** `+12.400ms`: the event's offset into its span, to the microsecond. */ +export function formatEventOffset(offsetMs: number): string { + const sign = offsetMs < 0 ? "-" : "+" + return `${sign}${formatDurationAtStep(Math.abs(offsetMs), 0.001)}` +} diff --git a/apps/web/src/routes/traces/$traceId.tsx b/apps/web/src/routes/traces/$traceId.tsx index a5494bb4f0..fe2a3e6738 100644 --- a/apps/web/src/routes/traces/$traceId.tsx +++ b/apps/web/src/routes/traces/$traceId.tsx @@ -1,7 +1,7 @@ import { warmAtoms } from "@effect-router/core" import * as React from "react" import { useNavigate, useRouterState, createFileRoute } from "@tanstack/react-router" -import { useAtomValue } from "@/lib/effect-atom" +import { Result, useAtomValue } from "@/lib/effect-atom" import { Option, Schema } from "effect" import { TraceId } from "@maple/domain" @@ -23,7 +23,10 @@ import { ResizablePanelGroup, ResizablePanel, ResizableHandle } from "@maple/ui/ import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@maple/ui/components/ui/sheet" import { useIsMobile } from "@maple/ui/hooks/use-media-query" import { type Span, type SpanNode, type SpanHierarchyResponse } from "@/api/warehouse/traces" -import { getSpanHierarchyResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" +import { + getSpanHierarchyResultAtom, + getTraceSpanEventsResultAtom, +} from "@/lib/services/atoms/warehouse-query-atoms" import { disabledResultAtom } from "@/lib/services/atoms/disabled-result-atom" import { findSpanById } from "@maple/ui/components/traces/flow-utils" import { HttpSpanLabel } from "@maple/ui/components/traces/http-span-label" @@ -265,6 +268,14 @@ function TraceDetailBody({ const isMobile = useIsMobile() + // Event markers arrive after the bars; until then (or on failure) the timeline draws none. + const spanEventsResult = useAtomValue( + getTraceSpanEventsResultAtom({ + data: { traceId: data.traceId, traceStartTime, totalDurationMs: data.totalDurationMs }, + }), + ) + const spanEvents = Result.isSuccess(spanEventsResult) ? spanEventsResult.value : undefined + // Shared by both layouts below so the tabs keep their state across a breakpoint change. const traceViewTabs = ( ) diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 4ace0d12d3..8c769345f4 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -111,6 +111,41 @@ export class SpanDetailRequest extends Schema.Class("SpanDeta endTime: Schema.optional(TinybirdDateTime), }) {} +/** One OTel span event; `exception` events carry `exception.*` attributes. */ +export const SpanEventSchema = Schema.Struct({ + name: Schema.String, + /** ISO-8601 UTC, millisecond precision. */ + timestamp: Schema.String, + /** Event time minus span start, in ms (sub-millisecond precision). */ + offsetMs: Schema.Number, + attributes: Schema.Record(Schema.String, Schema.String), +}) + +export class TraceSpanEventsRequest extends Schema.Class("TraceSpanEventsRequest")({ + traceId: TraceId, + /** The trace's own span window, so `traces` is read by its time range. */ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, +}) {} + +/** Event markers for one span of the trace timeline. */ +export const TimelineSpanEventSchema = Schema.Struct({ + name: Schema.String, + /** Event time minus span start, in ms. */ + offsetMs: Schema.Number, +}) + +export class TraceSpanEventsResponse extends Schema.Class("TraceSpanEventsResponse")( + { + data: Schema.Array( + Schema.Struct({ + spanId: SpanId, + events: Schema.Array(TimelineSpanEventSchema), + }), + ), + }, +) {} + export class SpanDetailResponse extends Schema.Class("SpanDetailResponse")({ data: Schema.NullOr( Schema.Struct({ @@ -118,6 +153,7 @@ export class SpanDetailResponse extends Schema.Class("SpanDe spanId: SpanId, spanAttributes: Schema.String, resourceAttributes: Schema.String, + events: Schema.optionalKey(Schema.Array(SpanEventSchema)), }), ), }) {} @@ -2602,6 +2638,13 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) + .add( + HttpApiEndpoint.post("traceSpanEvents", "/trace-span-events", { + payload: TraceSpanEventsRequest, + success: TraceSpanEventsResponse, + error: queryEngineEndpointErrors, + }), + ) .add( HttpApiEndpoint.post("errorsByType", "/errors-by-type", { payload: ErrorsByTypeRequest, diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index dbf0f28427..ecde1eeefe 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -835,12 +835,13 @@ SELECT GROUP BY fingerprintHash FORMAT JSON --- builder:errors:spanDetailQuery:default [02aceae6] +-- builder:errors:spanDetailQuery:default [a53b55ab] SELECT trace_detail_spans.TraceId AS traceId, trace_detail_spans.SpanId AS spanId, trace_detail_spans.ParentSpanId AS parentSpanId, if(((trace_detail_spans.SpanName LIKE 'http.server %' OR trace_detail_spans.SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (trace_detail_spans.SpanAttributes['http.route'] != '' OR trace_detail_spans.SpanAttributes['url.path'] != '')), concat(if(trace_detail_spans.SpanName LIKE 'http.server %', replaceOne(trace_detail_spans.SpanName, 'http.server ', ''), trace_detail_spans.SpanName), ' ', if(trace_detail_spans.SpanAttributes['http.route'] != '', trace_detail_spans.SpanAttributes['http.route'], trace_detail_spans.SpanAttributes['url.path'])), trace_detail_spans.SpanName) AS spanName, + trace_detail_spans.SpanName AS rawSpanName, trace_detail_spans.ServiceName AS serviceName, trace_detail_spans.SpanKind AS spanKind, trace_detail_spans.Duration / 1000000 AS durationMs, @@ -856,12 +857,13 @@ SELECT LIMIT 1 FORMAT JSON --- builder:errors:spanDetailQuery:narrowByTime [39008ef8] +-- builder:errors:spanDetailQuery:narrowByTime [695c65dd] SELECT trace_detail_spans.TraceId AS traceId, trace_detail_spans.SpanId AS spanId, trace_detail_spans.ParentSpanId AS parentSpanId, if(((trace_detail_spans.SpanName LIKE 'http.server %' OR trace_detail_spans.SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (trace_detail_spans.SpanAttributes['http.route'] != '' OR trace_detail_spans.SpanAttributes['url.path'] != '')), concat(if(trace_detail_spans.SpanName LIKE 'http.server %', replaceOne(trace_detail_spans.SpanName, 'http.server ', ''), trace_detail_spans.SpanName), ' ', if(trace_detail_spans.SpanAttributes['http.route'] != '', trace_detail_spans.SpanAttributes['http.route'], trace_detail_spans.SpanAttributes['url.path'])), trace_detail_spans.SpanName) AS spanName, + trace_detail_spans.SpanName AS rawSpanName, trace_detail_spans.ServiceName AS serviceName, trace_detail_spans.SpanKind AS spanKind, trace_detail_spans.Duration / 1000000 AS durationMs, @@ -879,6 +881,38 @@ SELECT LIMIT 1 FORMAT JSON +-- builder:errors:spanEventsByTraceQuery:default [bfef7dd0] +SELECT + traces.SpanId AS spanId, + toString(traces.Timestamp) AS spanStart, + toJSONString(traces.EventsTimestamp) AS eventsTimestamp, + toJSONString(traces.EventsName) AS eventsName + FROM traces + WHERE traces.OrgId = 'org_sql_catalog' + AND traces.Timestamp >= '2026-01-01 10:30:00' + AND traces.Timestamp <= '2026-01-03 14:15:00' + AND traces.TraceId = '0af7651916cd43dd8448eb211c80319c' + AND notEmpty(traces.EventsName) + LIMIT 5000 + FORMAT JSON + +-- builder:errors:spanEventsQuery:default [9e3ce44c] +SELECT + toString(traces.Timestamp) AS spanStart, + toJSONString(traces.EventsTimestamp) AS eventsTimestamp, + toJSONString(traces.EventsName) AS eventsName, + toJSONString(traces.EventsAttributes) AS eventsAttributes + FROM traces + WHERE traces.OrgId = 'org_sql_catalog' + AND traces.ServiceName = 'api' + AND traces.SpanName = 'GET' + AND traces.Timestamp >= '2026-01-01 10:30:00' + AND traces.Timestamp <= '2026-01-03 14:15:00' + AND traces.TraceId = '0af7651916cd43dd8448eb211c80319c' + AND traces.SpanId = 'b7ad6b7169203331' + LIMIT 1 + FORMAT JSON + -- builder:errors:tracesDurationStatsQuery:rollup [b66532bf] SELECT minIf(durationMin, traceCount > 0) / 1000000 AS minDurationMs, diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index 233ecbee4d..672f24fb9f 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -816,6 +816,29 @@ export const builderFixtures: ReadonlyArray = [ window, ), }, + { + // query-engine.http.ts spanDetail, after the detail row resolves the span's keys + module: "errors", + name: "spanEventsQuery", + label: "default", + compile: () => + CH.compileUnsafe( + CH.spanEventsQuery({ + traceId: TRACE_ID, + spanId: SPAN_ID, + serviceName: "api", + spanName: "GET", + }), + window, + ), + }, + { + // query-engine.http.ts traceSpanEvents, the trace timeline's event markers + module: "errors", + name: "spanEventsByTraceQuery", + label: "default", + compile: () => CH.compileUnsafe(CH.spanEventsByTraceQuery({ traceId: TRACE_ID }), window), + }, { // ErrorsService errorIssuesScan (the scheduled sweep) module: "errors", diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index a1ae824166..544dee7fee 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -9,6 +9,8 @@ import { tracesDurationStatsQuery, spanHierarchyQuery, spanDetailQuery, + spanEventsQuery, + spanEventsByTraceQuery, traceTimeProbeQuery, recentTraceTimeProbeQuery, } from "./queries/errors" @@ -944,6 +946,45 @@ describe("converted queries", () => { expect(sql).toContain("Timestamp <= '2026-04-15 15:00:00'") }) + it("spanEventsQuery seeks traces by the sorting-key prefix and returns the Events arrays", () => { + const q = spanEventsQuery({ traceId: "abc", spanId: "s1", serviceName: "api", spanName: "GET" }) + const { sql } = compileUnsafe(q, { + orgId: OrgId.make("org_1"), + startTime: "2026-04-15 13:00:00", + endTime: "2026-04-15 13:00:01", + }) + expect(sql).toContain("FROM traces") + expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("ServiceName = 'api'") + expect(sql).toContain("SpanName = 'GET'") + expect(sql).toContain("Timestamp >= '2026-04-15 13:00:00'") + expect(sql).toContain("Timestamp <= '2026-04-15 13:00:01'") + expect(sql).toContain("TraceId = 'abc'") + expect(sql).toContain("SpanId = 's1'") + expect(sql).toContain("toJSONString(traces.EventsTimestamp) AS eventsTimestamp") + expect(sql).toContain("toJSONString(traces.EventsName) AS eventsName") + expect(sql).toContain("toJSONString(traces.EventsAttributes) AS eventsAttributes") + expect(sql).toContain("LIMIT 1") + }) + + it("spanEventsByTraceQuery reads only spans with events inside the trace window", () => { + const q = spanEventsByTraceQuery({ traceId: "abc", limit: 10 }) + const { sql } = compileUnsafe(q, { + orgId: OrgId.make("org_1"), + startTime: "2026-04-15 13:00:00", + endTime: "2026-04-15 13:00:05", + }) + expect(sql).toContain("FROM traces") + expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("TraceId = 'abc'") + expect(sql).toContain("Timestamp >= '2026-04-15 13:00:00'") + expect(sql).toContain("Timestamp <= '2026-04-15 13:00:05'") + expect(sql).toContain("notEmpty(traces.EventsName)") + expect(sql).toContain("toJSONString(traces.EventsName) AS eventsName") + expect(sql).not.toContain("EventsAttributes") + expect(sql).toContain("LIMIT 10") + }) + it("traceTimeProbeQuery is a cheap LIMIT-1 timestamp seek with no ORDER BY", () => { const q = traceTimeProbeQuery({ traceId: "abc123" }) const { sql } = compileUnsafe(q, { orgId: OrgId.make("org_1") }) diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 76cd7dfdf6..ae083d8e94 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -344,6 +344,8 @@ export { spanHierarchyQuery, SPAN_HIERARCHY_MAX_SPANS, spanDetailQuery, + spanEventsQuery, + spanEventsByTraceQuery, traceTimeProbeQuery, recentTraceTimeProbeQuery, tracesDurationStatsQuery, @@ -378,6 +380,9 @@ export { type SpanHierarchyOutput, type SpanDetailOpts, type SpanDetailOutput, + type SpanEventsOpts, + type SpanEventsOutput, + type SpanEventsByTraceOutput, type TraceTimeProbeOutput, type TracesDurationStatsOpts, type TracesDurationStatsOutput, diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 690738d6c0..c226fc0669 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -8,7 +8,7 @@ import * as CH from "@maple-dev/effect-orm/expr" // From the root, not `/expr`: these overloads take a `CHQuery`, keeping the // subquery's params, table names and column types checked. import { exists, inSubquery } from "@maple-dev/effect-orm/clickhouse" -import { param } from "@maple-dev/effect-orm/clickhouse" +import { defineCondFn, param } from "@maple-dev/effect-orm/clickhouse" import { from, fromQuery, @@ -468,6 +468,8 @@ export interface SpanDetailOutput { readonly spanId: string readonly parentSpanId: string readonly spanName: string + /** The stored `SpanName`, before the HTTP display rewrite. Keys `spanEventsQuery`. */ + readonly rawSpanName: string readonly serviceName: string readonly spanKind: string readonly durationMs: number @@ -495,6 +497,7 @@ export function spanDetailQuery(opts: SpanDetailOpts) { $.SpanAttributes.get("http.route"), $.SpanAttributes.get("url.path"), ), + rawSpanName: $.SpanName, serviceName: $.ServiceName, spanKind: $.SpanKind, durationMs: $.Duration.div(1000000), @@ -515,6 +518,88 @@ export function spanDetailQuery(opts: SpanDetailOpts) { .format("JSON") } +// Span events — the OTel events (incl. `exception`) recorded on one span + +export interface SpanEventsOpts { + traceId: string + spanId: string + serviceName: string + /** The stored `SpanName` (`SpanDetailOutput.rawSpanName`), not the display name. */ + spanName: string +} + +export interface SpanEventsOutput { + /** The span's own start, rendered like the event timestamps so offsets keep ns precision. */ + readonly spanStart: string + readonly eventsTimestamp: string + readonly eventsName: string + readonly eventsAttributes: string +} + +/** + * Events for one span, read from `traces` because `trace_detail_spans` does not + * carry the Events arrays. `(OrgId, ServiceName, SpanName, toDateTime(Timestamp))` + * is the sorting key, so with the span's exact start second (`startTime` and + * `endTime` one second apart) this seeks a handful of granules. + */ +export function spanEventsQuery(opts: SpanEventsOpts) { + return from(Traces) + .select(($) => ({ + spanStart: CH.toString_($.Timestamp), + eventsTimestamp: CH.toJSONString($.EventsTimestamp), + eventsName: CH.toJSONString($.EventsName), + eventsAttributes: CH.toJSONString($.EventsAttributes), + })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.ServiceName.eq(opts.serviceName), + $.SpanName.eq(opts.spanName), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + $.TraceId.eq(opts.traceId), + $.SpanId.eq(opts.spanId), + ]) + .limit(1) + .format("JSON") +} + +// Span events by trace — event markers for every span of one trace + +export interface SpanEventsByTraceOutput { + readonly spanId: string + /** The span's own start, rendered like the event timestamps so offsets keep ns precision. */ + readonly spanStart: string + readonly eventsTimestamp: string + readonly eventsName: string +} + +const notEmpty = defineCondFn<[CH.Expr>]>("notEmpty") + +/** + * Event times and names for the spans of one trace that recorded any, for the + * timeline markers. Attributes stay on `spanEventsQuery`, loaded per span. + * `TraceId` is not in `traces`' sorting key; `idx_trace_id` and the trace's own + * `startTime`/`endTime` window keep this to a few granules. + */ +export function spanEventsByTraceQuery(opts: { traceId: string; limit?: number }) { + return from(Traces) + .select(($) => ({ + spanId: $.SpanId, + spanStart: CH.toString_($.Timestamp), + eventsTimestamp: CH.toJSONString($.EventsTimestamp), + eventsName: CH.toJSONString($.EventsName), + })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + $.TraceId.eq(opts.traceId), + notEmpty($.EventsName), + ]) + .limit(opts.limit ?? SPAN_HIERARCHY_MAX_SPANS) + .format("JSON") +} + // Trace timestamp probe — resolve any one span timestamp for a trace export interface TraceTimeProbeOutput { diff --git a/packages/query-engine/src/index.ts b/packages/query-engine/src/index.ts index 1fec482333..e53818ed53 100644 --- a/packages/query-engine/src/index.ts +++ b/packages/query-engine/src/index.ts @@ -11,6 +11,7 @@ export * from "./widget-request" export * from "./raw-sql-chart-rows" export * from "./route-rows" export * from "./traces-shared" +export * from "./span-events" export * from "./canonical-json" export * as CH from "./ch" export * as Observability from "./observability" diff --git a/packages/query-engine/src/observability/span-detail.test.ts b/packages/query-engine/src/observability/span-detail.test.ts index e8cdf0208c..537e734731 100644 --- a/packages/query-engine/src/observability/span-detail.test.ts +++ b/packages/query-engine/src/observability/span-detail.test.ts @@ -13,6 +13,7 @@ const detailRow = { spanId: SPAN_ID, parentSpanId: "", spanName: "GET /orders", + rawSpanName: "GET", serviceName: "api", spanKind: "Server", durationMs: 12, diff --git a/packages/query-engine/src/span-events.test.ts b/packages/query-engine/src/span-events.test.ts new file mode 100644 index 0000000000..6c95e8f7e1 --- /dev/null +++ b/packages/query-engine/src/span-events.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest" +import { parseSpanEvents } from "./span-events" + +const raw = (overrides: Partial[0]> = {}) => ({ + spanStart: "2026-03-20 08:00:00.000100000", + eventsTimestamp: JSON.stringify(["2026-03-20 08:00:01.250300000", "2026-03-20 08:00:00.000600000"]), + eventsName: JSON.stringify(["exception", "cache.miss"]), + eventsAttributes: JSON.stringify([{ "exception.type": "TypeError" }, { key: "user:1" }]), + ...overrides, +}) + +describe("parseSpanEvents", () => { + it("zips the parallel arrays and sorts by time", () => { + const events = parseSpanEvents(raw()) + expect(events.map((e) => e.name)).toEqual(["cache.miss", "exception"]) + expect(events[0]?.attributes).toEqual({ key: "user:1" }) + expect(events[1]?.attributes).toEqual({ "exception.type": "TypeError" }) + expect(events[1]?.timestamp).toBe("2026-03-20T08:00:01.250Z") + }) + + it("keeps sub-millisecond precision in offsets", () => { + const [first, second] = parseSpanEvents(raw()) + expect(first?.offsetMs).toBeCloseTo(0.5, 9) + expect(second?.offsetMs).toBeCloseTo(1250.2, 9) + }) + + it("tolerates missing names and attributes for an event", () => { + const events = parseSpanEvents( + raw({ eventsName: JSON.stringify(["only"]), eventsAttributes: JSON.stringify([]) }), + ) + expect(events.map((e) => [e.name, e.attributes])).toEqual([ + ["", {}], + ["only", {}], + ]) + }) + + it("reads times and names alone when attributes were not selected", () => { + const { eventsAttributes: _, ...withoutAttributes } = raw() + const events = parseSpanEvents(withoutAttributes) + expect(events.map((e) => [e.name, e.attributes])).toEqual([ + ["cache.miss", {}], + ["exception", {}], + ]) + }) + + it("returns no events for empty or undecodable arrays", () => { + expect(parseSpanEvents(raw({ eventsTimestamp: "[]" }))).toEqual([]) + expect(parseSpanEvents(raw({ eventsTimestamp: "not json" }))).toEqual([]) + }) +}) diff --git a/packages/query-engine/src/span-events.ts b/packages/query-engine/src/span-events.ts new file mode 100644 index 0000000000..80e2762b58 --- /dev/null +++ b/packages/query-engine/src/span-events.ts @@ -0,0 +1,77 @@ +// Span events: zip the warehouse's parallel Events arrays into records. +// +// `traces` stores OTel span events as three parallel columns (EventsTimestamp, +// EventsName, EventsAttributes). `spanEventsQuery` returns each as a JSON array +// string; this turns them into one record per event with its offset from the +// span start. Driver-free so the API, MCP and web can share it. + +import { Option, Schema } from "effect" +import { parseWarehouseDateTime, warehouseDateTimeToIso } from "./datetime" + +export interface SpanEvent { + readonly name: string + /** ISO-8601 UTC, millisecond precision. */ + readonly timestamp: string + /** Event time minus span start, in ms with sub-millisecond precision. */ + readonly offsetMs: number + readonly attributes: Readonly> +} + +export interface RawSpanEvents { + readonly spanStart: string + readonly eventsTimestamp: string + readonly eventsName: string + /** Absent when only times and names were read (timeline markers). */ + readonly eventsAttributes?: string +} + +const decodeStrings = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Array(Schema.String))) +const decodeAttributes = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Array(Schema.Record(Schema.String, Schema.String))), +) + +const DATETIME64_PATTERN = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?/ + +/** + * A DateTime64(9) string as whole epoch ms plus the nanoseconds past that ms. + * `Date.parse` alone would drop everything below a millisecond. + */ +const parseNanos = (value: string): Option.Option<{ readonly ms: number; readonly subMsNanos: number }> => { + const match = DATETIME64_PATTERN.exec(value.trim()) + if (!match) return Option.none() + const [, date, time, fraction = ""] = match + const seconds = parseWarehouseDateTime(`${date} ${time}`) + if (Number.isNaN(seconds)) return Option.none() + const nanos = Number(fraction.padEnd(9, "0")) + return Option.some({ ms: seconds + Math.floor(nanos / 1e6), subMsNanos: nanos % 1e6 }) +} + +/** + * Zip the parallel arrays into events sorted by time. Undecodable input yields + * no events rather than an error: events are supplementary to the span detail. + */ +export const parseSpanEvents = (raw: RawSpanEvents): ReadonlyArray => { + const timestamps = Option.getOrElse(decodeStrings(raw.eventsTimestamp), () => []) + const names = Option.getOrElse(decodeStrings(raw.eventsName), () => []) + const attributes = Option.getOrElse(decodeAttributes(raw.eventsAttributes), () => []) + const start = parseNanos(raw.spanStart) + + return timestamps + .flatMap((value, index) => + Option.match(parseNanos(value), { + onNone: () => [], + onSome: (at) => [ + { + name: names[index] ?? "", + timestamp: warehouseDateTimeToIso(value), + offsetMs: Option.match(start, { + onNone: () => 0, + onSome: (s) => at.ms - s.ms + (at.subMsNanos - s.subMsNanos) / 1e6, + }), + attributes: attributes[index] ?? {}, + }, + ], + }), + ) + .sort((a, b) => a.offsetMs - b.offsetMs) +} diff --git a/packages/ui/src/components/traces/trace-timeline-row.tsx b/packages/ui/src/components/traces/trace-timeline-row.tsx index 64b8ae6776..36a11a024f 100644 --- a/packages/ui/src/components/traces/trace-timeline-row.tsx +++ b/packages/ui/src/components/traces/trace-timeline-row.tsx @@ -5,12 +5,14 @@ import { cn } from "../../lib/utils" import { getServiceColor } from "../../lib/colors" import { countLabel, formatDuration } from "../../lib/format" import { describeSpan } from "../../lib/span-category" +import { spanEventMarkers, type TimelineSpanEvent } from "../../lib/span-event-markers" +import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip" import type { TimelineBar } from "./trace-timeline-types" import { DEPTH_INDENT, ROW_HEIGHT } from "./trace-timeline-types" /** Indent stops growing past this depth so deep traces keep room for the label. */ const MAX_INDENT_DEPTH = 12 -import { TONE_TEXT } from "../../lib/tone" +import { TONE_FILL, TONE_TEXT } from "../../lib/tone" interface TraceTimelineRowProps { bar: TimelineBar @@ -25,6 +27,8 @@ interface TraceTimelineRowProps { matched: boolean /** A single-service trace repeats one name down every row — the header already said it. */ showService: boolean + /** Span events, drawn as ticks along the bar. */ + events?: ReadonlyArray onSelect: (spanId: string) => void /** `wholeSubtree` comes from an Alt/Option-click. */ onToggleCollapse: (spanId: string, wholeSubtree: boolean) => void @@ -56,6 +60,7 @@ function TraceTimelineRowImpl({ dimmed, matched, showService, + events, onSelect, onToggleCollapse, onZoomSpan, @@ -214,6 +219,14 @@ function TraceTimelineRowImpl({ + {events && events.length > 0 && ( + + )} + {/* Outside label for bars too narrow to hold one. Sits on the right by default; the decoration pass flips it left near the right edge of the column. */} + durationMs: number + onHover: TraceTimelineRowProps["onHover"] +}) { + const markers = React.useMemo(() => spanEventMarkers(events, durationMs), [events, durationMs]) + return markers.map((marker, index) => ( + + onHover(null, null)} + onMouseMove={(e) => e.stopPropagation()} + /> + } + > + + + + + {marker.name || "(unnamed event)"} + {" "} + + +{formatDuration(marker.offsetMs)} + + + + )) +} + export const TraceTimelineRow = React.memo(TraceTimelineRowImpl) diff --git a/packages/ui/src/components/traces/trace-timeline.tsx b/packages/ui/src/components/traces/trace-timeline.tsx index b1d60aa958..44cbf76368 100644 --- a/packages/ui/src/components/traces/trace-timeline.tsx +++ b/packages/ui/src/components/traces/trace-timeline.tsx @@ -61,6 +61,7 @@ export function TraceTimeline() { onSelectSpan, colorBy, setColorBy, + spanEvents, } = useTraceView() // Surfaced in the toolbar: a correction the user cannot see in the bars themselves // should still be visible as a fact about the trace. @@ -459,9 +460,7 @@ export function TraceTimeline() { }, [virtualRangeKey, repaintRowDecorations]) if (rootSpans.length === 0) { - return ( - No spans found for this trace - ) + return No spans found for this trace } return ( @@ -633,6 +632,7 @@ export function TraceTimeline() { dimmed={isSearchActive && !matched} matched={matched} showService={services.length > 1} + events={spanEvents?.get(id)} onSelect={handleSelect} onToggleCollapse={handleToggleCollapse} onZoomSpan={handleZoomSpan} @@ -667,32 +667,20 @@ export function TraceTimeline() { {/* Pointer/keyboard hints are meaningless on touch — the legend takes the whole bar there. */}
    - - Drag - {" "} - zoom + Drag zoom - - W A S D - {" "} - navigate + W A S D navigate - - / - {" "} - search + / search
    {/* Legend: scrolls sideways on narrow screens rather than wrapping the bar taller. */} diff --git a/packages/ui/src/components/traces/trace-view-context.tsx b/packages/ui/src/components/traces/trace-view-context.tsx index c140425f66..ad221c0526 100644 --- a/packages/ui/src/components/traces/trace-view-context.tsx +++ b/packages/ui/src/components/traces/trace-view-context.tsx @@ -2,6 +2,7 @@ import * as React from "react" import type { ReactNode } from "react" import type { SpanNode } from "../../lib/types" import type { ColorByField } from "./color-by" +import type { SpanEventsBySpanId } from "../../lib/span-event-markers" interface TraceViewContextValue { rootSpans: SpanNode[] @@ -12,6 +13,8 @@ interface TraceViewContextValue { onSelectSpan?: (span: SpanNode) => void colorBy: ColorByField setColorBy: (next: ColorByField) => void + /** Loaded after the hierarchy; absent until then, and for views that never fetch them. */ + spanEvents?: SpanEventsBySpanId } const TraceViewContext = React.createContext(null) @@ -28,6 +31,7 @@ export function TraceViewProvider({ children, ...value }: TraceViewContextValue value.onSelectSpan, value.colorBy, value.setColorBy, + value.spanEvents, ], ) return {children} diff --git a/packages/ui/src/components/traces/trace-view-tabs.tsx b/packages/ui/src/components/traces/trace-view-tabs.tsx index 217d4baa6f..26f218ebad 100644 --- a/packages/ui/src/components/traces/trace-view-tabs.tsx +++ b/packages/ui/src/components/traces/trace-view-tabs.tsx @@ -8,6 +8,7 @@ import { TraceFlowView } from "./flow-view" import { TraceViewProvider } from "./trace-view-context" import { DEFAULT_COLOR_BY, type ColorByField } from "./color-by" import type { SpanNode, Span } from "../../lib/types" +import type { SpanEventsBySpanId } from "../../lib/span-event-markers" export type TraceView = "waterfall" | "timeline" | "flow" @@ -26,6 +27,8 @@ interface TraceViewTabsProps { /** Controlled active tab. Omit to let the tabs own it, starting on the timeline. */ view?: TraceView onViewChange?: (view: TraceView) => void + /** Event markers for the timeline bars. */ + spanEvents?: SpanEventsBySpanId } export function TraceViewTabs({ @@ -38,6 +41,7 @@ export function TraceViewTabs({ onSelectSpan, view, onViewChange, + spanEvents, }: TraceViewTabsProps) { // _spans is reserved for future Flow view implementation const [colorBy, setColorBy] = React.useState(DEFAULT_COLOR_BY) @@ -52,6 +56,7 @@ export function TraceViewTabs({ onSelectSpan={onSelectSpan} colorBy={colorBy} setColorBy={setColorBy} + spanEvents={spanEvents} > { + it("places each event at its share of the span duration", () => { + const markers = spanEventMarkers( + [ + { name: "cache.miss", offsetMs: 25 }, + { name: "exception", offsetMs: 75 }, + ], + 100, + ) + expect(markers.map((m) => [m.name, m.percent, m.isException])).toEqual([ + ["cache.miss", 25, false], + ["exception", 75, true], + ]) + }) + + it("pins events outside the span to the nearest edge", () => { + const markers = spanEventMarkers( + [ + { name: "early", offsetMs: -3 }, + { name: "late", offsetMs: 140 }, + ], + 100, + ) + expect(markers.map((m) => m.percent)).toEqual([0, 100]) + expect(markers.map((m) => m.offsetMs)).toEqual([-3, 140]) + }) + + it("puts every event at the start of a zero-duration span", () => { + expect(spanEventMarkers([{ name: "log", offsetMs: 0.2 }], 0).map((m) => m.percent)).toEqual([0]) + }) +}) diff --git a/packages/ui/src/lib/span-event-markers.ts b/packages/ui/src/lib/span-event-markers.ts new file mode 100644 index 0000000000..d7040b5fb0 --- /dev/null +++ b/packages/ui/src/lib/span-event-markers.ts @@ -0,0 +1,35 @@ +// Span event markers on the trace timeline: where along its bar each event sits. + +/** One OTel span event as the timeline needs it: no attributes. */ +export interface TimelineSpanEvent { + readonly name: string + /** Event time minus span start, in ms. */ + readonly offsetMs: number +} + +/** Timeline events keyed by span id; spans without events are absent. */ +export type SpanEventsBySpanId = ReadonlyMap> + +export interface SpanEventMarker { + readonly name: string + readonly offsetMs: number + /** Position along the bar, 0-100. */ + readonly percent: number + readonly isException: boolean +} + +/** + * Place events along a span's bar. Offsets outside the span (clock skew, or an + * event recorded after `end()`) pin to the nearest edge rather than escaping it. + */ +export function spanEventMarkers( + events: ReadonlyArray, + durationMs: number, +): ReadonlyArray { + return events.map((event) => ({ + name: event.name, + offsetMs: event.offsetMs, + percent: durationMs > 0 ? Math.min(100, Math.max(0, (event.offsetMs / durationMs) * 100)) : 0, + isException: event.name === "exception", + })) +} From 63b201cf9eb50b90d85db2f5ab17f984444d40a4 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 16 Oct 2026 01:06:43 +0200 Subject: [PATCH 2/4] fix(traces): place event markers on the time axis, order the trace events cap - Event markers now sit on the trace's time axis with the bar's --vp0/--vpk transform instead of a percent of the bar box, which is clamped to the view when zoomed, so they no longer drift off their time. - spanEventsByTraceQuery orders by span start before its LIMIT, like spanHierarchyQuery, so the cap keeps the spans the timeline shows. - formatEventOffset keeps microseconds for offsets of a second or more. --- apps/web/src/lib/traces/span-events.test.ts | 3 + apps/web/src/lib/traces/span-events.ts | 5 +- .../src/__sql_baseline__/catalog.sql | 3 +- packages/query-engine/src/ch/ch.test.ts | 1 + .../query-engine/src/ch/queries/errors.ts | 37 ++++--- .../components/traces/trace-timeline-row.tsx | 103 ++++++++++-------- .../lib/__tests__/span-event-markers.test.ts | 21 +++- packages/ui/src/lib/span-event-markers.ts | 12 +- 8 files changed, 113 insertions(+), 72 deletions(-) diff --git a/apps/web/src/lib/traces/span-events.test.ts b/apps/web/src/lib/traces/span-events.test.ts index 1e27119f2d..fac6a9b1cc 100644 --- a/apps/web/src/lib/traces/span-events.test.ts +++ b/apps/web/src/lib/traces/span-events.test.ts @@ -46,5 +46,8 @@ describe("formatEventOffset", () => { expect(formatEventOffset(0.25)).toBe("+250μs") expect(formatEventOffset(-1.5)).toBe("-1.500ms") expect(formatEventOffset(2500)).toBe("+2.500s") + expect(formatEventOffset(1000)).toBe("+1.000s") + expect(formatEventOffset(1000.123)).toBe("+1.000123s") + expect(formatEventOffset(-61_000.5)).toBe("-61.0005s") }) }) diff --git a/apps/web/src/lib/traces/span-events.ts b/apps/web/src/lib/traces/span-events.ts index c01de06452..0cf435441c 100644 --- a/apps/web/src/lib/traces/span-events.ts +++ b/apps/web/src/lib/traces/span-events.ts @@ -37,5 +37,8 @@ export function describeSpanEvent(event: SpanEvent): DescribedSpanEvent { /** `+12.400ms`: the event's offset into its span, to the microsecond. */ export function formatEventOffset(offsetMs: number): string { const sign = offsetMs < 0 ? "-" : "+" - return `${sign}${formatDurationAtStep(Math.abs(offsetMs), 0.001)}` + const magnitude = Math.abs(offsetMs) + // Past a second the shared formatter rounds to the millisecond; keep the microseconds. + if (magnitude >= 1000) return `${sign}${(magnitude / 1000).toFixed(6).replace(/0{1,3}$/, "")}s` + return `${sign}${formatDurationAtStep(magnitude, 0.001)}` } diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index ecde1eeefe..febee4074e 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -881,7 +881,7 @@ SELECT LIMIT 1 FORMAT JSON --- builder:errors:spanEventsByTraceQuery:default [bfef7dd0] +-- builder:errors:spanEventsByTraceQuery:default [b03e077c] SELECT traces.SpanId AS spanId, toString(traces.Timestamp) AS spanStart, @@ -893,6 +893,7 @@ SELECT AND traces.Timestamp <= '2026-01-03 14:15:00' AND traces.TraceId = '0af7651916cd43dd8448eb211c80319c' AND notEmpty(traces.EventsName) + ORDER BY spanStart ASC LIMIT 5000 FORMAT JSON diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index 544dee7fee..91e794661b 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -980,6 +980,7 @@ describe("converted queries", () => { expect(sql).toContain("Timestamp >= '2026-04-15 13:00:00'") expect(sql).toContain("Timestamp <= '2026-04-15 13:00:05'") expect(sql).toContain("notEmpty(traces.EventsName)") + expect(sql).toMatch(/ORDER BY spanStart ASC\s+LIMIT 10/) expect(sql).toContain("toJSONString(traces.EventsName) AS eventsName") expect(sql).not.toContain("EventsAttributes") expect(sql).toContain("LIMIT 10") diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index c226fc0669..1d0885bd9a 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -582,22 +582,27 @@ const notEmpty = defineCondFn<[CH.Expr>]>("notEmpty") * `startTime`/`endTime` window keep this to a few granules. */ export function spanEventsByTraceQuery(opts: { traceId: string; limit?: number }) { - return from(Traces) - .select(($) => ({ - spanId: $.SpanId, - spanStart: CH.toString_($.Timestamp), - eventsTimestamp: CH.toJSONString($.EventsTimestamp), - eventsName: CH.toJSONString($.EventsName), - })) - .where(($) => [ - $.OrgId.eq(orgIdParam), - $.Timestamp.gte(param.dateTime("startTime")), - $.Timestamp.lte(param.dateTime("endTime")), - $.TraceId.eq(opts.traceId), - notEmpty($.EventsName), - ]) - .limit(opts.limit ?? SPAN_HIERARCHY_MAX_SPANS) - .format("JSON") + return ( + from(Traces) + .select(($) => ({ + spanId: $.SpanId, + spanStart: CH.toString_($.Timestamp), + eventsTimestamp: CH.toJSONString($.EventsTimestamp), + eventsName: CH.toJSONString($.EventsName), + })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.Timestamp.gte(param.dateTime("startTime")), + $.Timestamp.lte(param.dateTime("endTime")), + $.TraceId.eq(opts.traceId), + notEmpty($.EventsName), + ]) + // Earliest first, like `spanHierarchyQuery`, so the cap keeps the spans the timeline shows. + // `spanStart` is fixed-width DateTime64 text, so it sorts in time order. + .orderBy(["spanStart", "asc"]) + .limit(opts.limit ?? SPAN_HIERARCHY_MAX_SPANS) + .format("JSON") + ) } // Trace timestamp probe — resolve any one span timestamp for a trace diff --git a/packages/ui/src/components/traces/trace-timeline-row.tsx b/packages/ui/src/components/traces/trace-timeline-row.tsx index 36a11a024f..0c125b25a9 100644 --- a/packages/ui/src/components/traces/trace-timeline-row.tsx +++ b/packages/ui/src/components/traces/trace-timeline-row.tsx @@ -219,14 +219,6 @@ function TraceTimelineRowImpl({     - {events && events.length > 0 && ( - - )} - {/* Outside label for bars too narrow to hold one. Sits on the right by default; the decoration pass flips it left near the right edge of the column. */} + {events && events.length > 0 && ( + + )} + {/* Clipping chevrons, pinned to the cell edges (Jaeger). Toggled by the decoration pass — CSS can't tell whether the clamp above actually bit. */} + spanStartMs: number durationMs: number onHover: TraceTimelineRowProps["onHover"] }) { - const markers = React.useMemo(() => spanEventMarkers(events, durationMs), [events, durationMs]) - return markers.map((marker, index) => ( - - onHover(null, null)} - onMouseMove={(e) => e.stopPropagation()} - /> - } - > - - - - - {marker.name || "(unnamed event)"} - {" "} - - +{formatDuration(marker.offsetMs)} - - - - )) + const markers = React.useMemo( + () => spanEventMarkers(events, spanStartMs, durationMs), + [events, spanStartMs, durationMs], + ) + return ( +
    + {markers.map((marker, index) => ( + + onHover(null, null)} + onMouseMove={(e) => e.stopPropagation()} + /> + } + > + + + + + {marker.name || "(unnamed event)"} + {" "} + + +{formatDuration(marker.offsetMs)} + + + + ))} +
    + ) } export const TraceTimelineRow = React.memo(TraceTimelineRowImpl) diff --git a/packages/ui/src/lib/__tests__/span-event-markers.test.ts b/packages/ui/src/lib/__tests__/span-event-markers.test.ts index 67b0cd582e..a563937a5a 100644 --- a/packages/ui/src/lib/__tests__/span-event-markers.test.ts +++ b/packages/ui/src/lib/__tests__/span-event-markers.test.ts @@ -2,33 +2,42 @@ import { describe, expect, it } from "vitest" import { spanEventMarkers } from "../span-event-markers" describe("spanEventMarkers", () => { - it("places each event at its share of the span duration", () => { + it("places each event at its time on the trace axis", () => { const markers = spanEventMarkers( [ { name: "cache.miss", offsetMs: 25 }, { name: "exception", offsetMs: 75 }, ], + 1000, 100, ) - expect(markers.map((m) => [m.name, m.percent, m.isException])).toEqual([ - ["cache.miss", 25, false], - ["exception", 75, true], + expect(markers.map((m) => [m.name, m.atMs, m.isException])).toEqual([ + ["cache.miss", 1025, false], + ["exception", 1075, true], ]) }) + it("keeps the event time of a span that runs far past the visible window", () => { + // Zoomed in on 0-10 ms: the bar of this 0-10 s span is clamped to the view, the + // marker must still sit at 5 ms, not at the middle of the clamped box. + const [marker] = spanEventMarkers([{ name: "retry", offsetMs: 5 }], 0, 10_000) + expect(marker?.atMs).toBe(5) + }) + it("pins events outside the span to the nearest edge", () => { const markers = spanEventMarkers( [ { name: "early", offsetMs: -3 }, { name: "late", offsetMs: 140 }, ], + 1000, 100, ) - expect(markers.map((m) => m.percent)).toEqual([0, 100]) + expect(markers.map((m) => m.atMs)).toEqual([1000, 1100]) expect(markers.map((m) => m.offsetMs)).toEqual([-3, 140]) }) it("puts every event at the start of a zero-duration span", () => { - expect(spanEventMarkers([{ name: "log", offsetMs: 0.2 }], 0).map((m) => m.percent)).toEqual([0]) + expect(spanEventMarkers([{ name: "log", offsetMs: 0.2 }], 40, 0).map((m) => m.atMs)).toEqual([40]) }) }) diff --git a/packages/ui/src/lib/span-event-markers.ts b/packages/ui/src/lib/span-event-markers.ts index d7040b5fb0..ccc7badfea 100644 --- a/packages/ui/src/lib/span-event-markers.ts +++ b/packages/ui/src/lib/span-event-markers.ts @@ -13,23 +13,25 @@ export type SpanEventsBySpanId = ReadonlyMap, + spanStartMs: number, durationMs: number, ): ReadonlyArray { return events.map((event) => ({ name: event.name, offsetMs: event.offsetMs, - percent: durationMs > 0 ? Math.min(100, Math.max(0, (event.offsetMs / durationMs) * 100)) : 0, + atMs: spanStartMs + Math.min(Math.max(durationMs, 0), Math.max(0, event.offsetMs)), isException: event.name === "exception", })) } From 71f4f02bf868ea1c43ae97a50f1b441624670b4f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 16 Oct 2026 01:14:05 +0200 Subject: [PATCH 3/4] fix(traces): sign negative event offsets, focusable markers, ai fixtures - Marker labels render a pre-span event as `-3.0ms`, not `+-3.0ms`. - Event markers are tab-focusable so keyboard users reach their tooltip, and unnamed events get a label. - inspect_span eval fixtures carry spanDetailQuery's new rawSpanName column; without it the row failed the compiled query's schema. --- apps/ai/src/mcp/__evals__/fixtures.ts | 2 ++ packages/ui/src/components/traces/trace-timeline-row.tsx | 5 +++-- packages/ui/src/lib/__tests__/span-event-markers.test.ts | 1 + packages/ui/src/lib/span-event-markers.ts | 5 +++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/ai/src/mcp/__evals__/fixtures.ts b/apps/ai/src/mcp/__evals__/fixtures.ts index 2066ee771a..7c4822ac78 100644 --- a/apps/ai/src/mcp/__evals__/fixtures.ts +++ b/apps/ai/src/mcp/__evals__/fixtures.ts @@ -112,6 +112,7 @@ export const makeSpanDetailRows = (): ReadonlyArray> => // the compiled query's schema now, so a partial one is a decode failure. parentSpanId: "", spanName: "POST /api/checkout", + rawSpanName: "POST", serviceName: FIXTURES.service, spanKind: "Server", durationMs: 120, @@ -168,6 +169,7 @@ export const makeAiSpanDetailRows = ( spanId, parentSpanId: "", spanName: "chat gpt-5", + rawSpanName: "chat gpt-5", serviceName: FIXTURES.service, spanKind: "Client", durationMs: 1_200, diff --git a/packages/ui/src/components/traces/trace-timeline-row.tsx b/packages/ui/src/components/traces/trace-timeline-row.tsx index 0c125b25a9..df14694b55 100644 --- a/packages/ui/src/components/traces/trace-timeline-row.tsx +++ b/packages/ui/src/components/traces/trace-timeline-row.tsx @@ -297,7 +297,8 @@ function SpanEventMarkers({ render={ {" "} - +{formatDuration(marker.offsetMs)} + {marker.offsetLabel} diff --git a/packages/ui/src/lib/__tests__/span-event-markers.test.ts b/packages/ui/src/lib/__tests__/span-event-markers.test.ts index a563937a5a..8d2da2a339 100644 --- a/packages/ui/src/lib/__tests__/span-event-markers.test.ts +++ b/packages/ui/src/lib/__tests__/span-event-markers.test.ts @@ -35,6 +35,7 @@ describe("spanEventMarkers", () => { ) expect(markers.map((m) => m.atMs)).toEqual([1000, 1100]) expect(markers.map((m) => m.offsetMs)).toEqual([-3, 140]) + expect(markers.map((m) => m.offsetLabel)).toEqual(["-3.0ms", "+140.0ms"]) }) it("puts every event at the start of a zero-duration span", () => { diff --git a/packages/ui/src/lib/span-event-markers.ts b/packages/ui/src/lib/span-event-markers.ts index ccc7badfea..dcb15be32b 100644 --- a/packages/ui/src/lib/span-event-markers.ts +++ b/packages/ui/src/lib/span-event-markers.ts @@ -1,5 +1,7 @@ // Span event markers on the trace timeline: where along its bar each event sits. +import { formatDuration } from "./format" + /** One OTel span event as the timeline needs it: no attributes. */ export interface TimelineSpanEvent { readonly name: string @@ -15,6 +17,8 @@ export interface SpanEventMarker { readonly offsetMs: number /** Trace-relative time the marker sits at, like the bar's `--b0`/`--b1`; kept inside the span. */ readonly atMs: number + /** `+12ms`, or `-3ms` for an event stamped before its span started (clock skew). */ + readonly offsetLabel: string readonly isException: boolean } @@ -32,6 +36,7 @@ export function spanEventMarkers( name: event.name, offsetMs: event.offsetMs, atMs: spanStartMs + Math.min(Math.max(durationMs, 0), Math.max(0, event.offsetMs)), + offsetLabel: `${event.offsetMs < 0 ? "-" : "+"}${formatDuration(Math.abs(event.offsetMs))}`, isException: event.name === "exception", })) } From 892d6277ca1c4c90b291f6c8071a4b89f9814b7d Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 16 Oct 2026 01:23:00 +0200 Subject: [PATCH 4/4] chore(lab): span events on the timeline lab fixture Markers on three spans (plain events, an exception, a pre-span event) and the Events panel for the selected span, so /lab/timeline shows the feature without a seeded warehouse. --- apps/web/src/lab/registry.ts | 2 +- apps/web/src/lab/timeline-fixture.ts | 48 +++++++++++++++++++++++++++- apps/web/src/lab/timeline-lab.tsx | 36 ++++++++++++++------- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/apps/web/src/lab/registry.ts b/apps/web/src/lab/registry.ts index 25032f298a..62593a6cb3 100644 --- a/apps/web/src/lab/registry.ts +++ b/apps/web/src/lab/registry.ts @@ -87,7 +87,7 @@ export const LAB_ENTRIES: ReadonlyArray = [ path: "/lab/timeline", title: "Trace timeline", description: - "The trace view tabs over a synthetic trace with a wide dynamic range — a 2s root down to sub-millisecond spans that only resolve at deep zoom.", + "The trace view tabs over a synthetic trace with a wide dynamic range — a 2s root down to sub-millisecond spans that only resolve at deep zoom, with span event markers and the events panel.", kind: "lab", session: "none", }, diff --git a/apps/web/src/lab/timeline-fixture.ts b/apps/web/src/lab/timeline-fixture.ts index 717ee505ca..39d136acd4 100644 --- a/apps/web/src/lab/timeline-fixture.ts +++ b/apps/web/src/lab/timeline-fixture.ts @@ -1,5 +1,6 @@ import { buildTraceDetail, type SpanHierarchyRow } from "@maple/ui/lib/span-tree" -import { formatWarehouseDateTimeMs } from "@maple/query-engine" +import { formatWarehouseDateTimeMs, type SpanEvent } from "@maple/query-engine" +import type { SpanEventsBySpanId } from "@maple/ui/lib/span-event-markers" const T0_MS = new Date("2026-07-21T10:00:00.000Z").getTime() @@ -65,3 +66,48 @@ const ROWS: SpanHierarchyRow[] = [ /** Synthetic trace for the timeline lab: 2s root, mid-size children, sub-ms fan. */ export const TIMELINE_LAB_TRACE = buildTraceDetail(ROWS) + +const event = ( + spanStartMs: number, + offsetMs: number, + name: string, + attributes: Record = {}, +): SpanEvent => ({ + name, + timestamp: new Date(T0_MS + spanStartMs + offsetMs).toISOString(), + offsetMs, + attributes, +}) + +/** OTel events on a few spans: plain events, an exception, one stamped before its span (clock skew). */ +export const TIMELINE_LAB_EVENTS: ReadonlyMap> = new Map([ + [ + "root", + [ + event(0, 4.2, "cache.miss", { "cache.key": "cart:8f2c" }), + event(0, 1210.0004, "retry", { "retry.attempt": "2", "retry.reason": "upstream 503" }), + ], + ], + ["svc0", [event(20, 1.75, "db.connection.acquired", { "db.pool.wait_ms": "1.7" })]], + [ + "svc3", + [ + event(920, -0.6, "request.received"), + event(920, 180.25, "exception", { + "exception.type": "SearchTimeoutError", + "exception.message": "search backend did not answer within 150ms", + "exception.stacktrace": + "SearchTimeoutError: search backend did not answer within 150ms\n at query (search/client.ts:88:11)\n at handler (search/route.ts:41:5)", + "exception.escaped": "true", + }), + ], + ], +]) + +/** The timeline only needs names and offsets. */ +export const TIMELINE_LAB_MARKERS: SpanEventsBySpanId = new Map( + [...TIMELINE_LAB_EVENTS].map(([spanId, events]) => [ + spanId, + events.map(({ name, offsetMs }) => ({ name, offsetMs })), + ]), +) diff --git a/apps/web/src/lab/timeline-lab.tsx b/apps/web/src/lab/timeline-lab.tsx index dc693fd77b..389444ca7e 100644 --- a/apps/web/src/lab/timeline-lab.tsx +++ b/apps/web/src/lab/timeline-lab.tsx @@ -3,21 +3,35 @@ import { useState } from "react" import { TraceViewTabs } from "@maple/ui/components/traces/trace-view-tabs" import type { SpanNode } from "@maple/ui/lib/types" -import { TIMELINE_LAB_TRACE as DETAIL } from "@/lab/timeline-fixture" +import { SpanEventsSection } from "@/components/traces/span-events-section" +import { + TIMELINE_LAB_TRACE as DETAIL, + TIMELINE_LAB_EVENTS, + TIMELINE_LAB_MARKERS, +} from "@/lab/timeline-fixture" export function TimelineLab() { const [selected, setSelected] = useState(undefined) + const events = selected ? TIMELINE_LAB_EVENTS.get(selected.spanId) : undefined return ( -
    - +
    +
    + +
    + {events && ( + + )}
    ) }