feat(traces): show span events in the span panel and on the timeline

apps/ai/src/mcp/__evals__/fixtures.ts

@@ -112,6 +112,7 @@ export const makeSpanDetailRows = (): ReadonlyArray<Record<string, unknown>> =>
 		// the compiled query's schema now, so a partial one is a decode failure.
 		parentSpanId: "",
 		spanName: "POST /api/checkout",
+		rawSpanName: "POST",
 		serviceName: FIXTURES.service,
 		spanKind: "Server",
 		durationMs: 120,
@@ -168,6 +169,7 @@ export const makeAiSpanDetailRows = (
 		spanId,
 		parentSpanId: "",
 		spanName: "chat gpt-5",
+		rawSpanName: "chat gpt-5",
 		serviceName: FIXTURES.service,
 		spanKind: "Client",
 		durationMs: 1_200,

apps/api/src/routes/internal/query-engine.http.ts

@@ -7,6 +7,7 @@ import {
 	type RawSqlValidationError,
 	SpanHierarchyResponse,
 	SpanDetailResponse,
+	TraceSpanEventsResponse,
 	ErrorsByTypeResponse,
 	ErrorsSparkResponse,
 	ErrorsSummaryResponse,
@@ -114,6 +115,7 @@ import {
 	computeBucketSecondsForRange,
 	formatWarehouseDateTime,
 	QueryEngineExecuteBatchResponse,
+	parseSpanEvents,
 	parseWarehouseDateTime,
 } from "@maple/query-engine"
 
@@ -517,14 +519,48 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query
 					Effect.gen(function* () {
 						const tenant = yield* CurrentTenant.Context
 						const row = yield* runQueryFirst(Queries.spanDetail, tenant, payload)
+						if (!row) return new SpanDetailResponse({ data: null })
+						// `trace_detail_spans` has no Events columns; read them off `traces`
+						// by the sorting key the detail row just resolved.
+						const startMs = DateTime.toEpochMillis(row.startTime)
+						const events = yield* runQueryFirst(Queries.spanEvents, tenant, {
+							traceId: row.traceId,
+							spanId: row.spanId,
+							serviceName: row.serviceName,
+							spanName: row.rawSpanName,
+							startTime: formatWarehouseDateTime(startMs),
+							endTime: formatWarehouseDateTime(startMs + 1000),
+						}).pipe(
+							Effect.map((raw) => (raw ? parseSpanEvents(raw) : [])),
+							// Events are supplementary; a failed lookup must not hide the attributes.
+							Effect.tapError((error) => Effect.logWarning("span events lookup failed", error)),
+							Effect.orElseSucceed(() => []),
+						)
 						return new SpanDetailResponse({
-							data: row
-								? {
-										...row,
-										traceId: decodeTraceId(row.traceId),
-										spanId: decodeSpanId(row.spanId),
-									}
-								: null,
+							data: {
+								traceId: decodeTraceId(row.traceId),
+								spanId: decodeSpanId(row.spanId),
+								spanAttributes: row.spanAttributes,
+								resourceAttributes: row.resourceAttributes,
+								events,
+							},
+						})
+					}),
+				)
+				.handle("traceSpanEvents", ({ payload }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						const rows = yield* runQuery(Queries.traceSpanEvents, tenant, payload)
+						return new TraceSpanEventsResponse({
+							data: rows.flatMap((row) => {
+								const events = parseSpanEvents(row).map(({ name, offsetMs }) => ({
+									name,
+									offsetMs,
+								}))
+								return events.length === 0
+									? []
+									: [{ spanId: decodeSpanId(row.spanId), events }]
+							}),
 						})
 					}),
 				)

apps/api/src/routes/queries.ts

@@ -24,6 +24,7 @@ import type {
 	RailwayInfraServiceTimeseriesRequest,
 	ServiceCloudflareStatsRequest,
 	SpanDetailRequest,
+	TraceSpanEventsRequest,
 	WorkloadInfraTimeseriesRequest,
 } from "@maple/domain/http"
 import {
@@ -673,6 +674,50 @@ export const Queries = {
 		},
 	}),
 
+	/**
+	 * Events of a span `spanDetail` already resolved. Keyed by the span's exact
+	 * start second, so the row it reads is written and immutable.
+	 */
+	spanEvents: defineQuery({
+		id: "spanEvents",
+		profile: "discovery",
+		cache: 600,
+		compile: (
+			payload: {
+				readonly traceId: string
+				readonly spanId: string
+				readonly serviceName: string
+				readonly spanName: string
+				readonly startTime: string
+				readonly endTime: string
+			},
+			orgId: OrgId,
+		) =>
+			CH.compile(
+				CH.spanEventsQuery({
+					traceId: payload.traceId,
+					spanId: payload.spanId,
+					serviceName: payload.serviceName,
+					spanName: payload.spanName,
+				}),
+				{ orgId, startTime: payload.startTime, endTime: payload.endTime },
+			),
+	}),
+
+	/** Event times and names for every span of a trace, for the timeline markers. */
+	traceSpanEvents: defineQuery({
+		id: "traceSpanEvents",
+		profile: "list",
+		cache: (payload: TraceSpanEventsRequest, nowMs: number) =>
+			traceCacheTtlSeconds(payload.endTime, nowMs),
+		compile: (payload: TraceSpanEventsRequest, orgId: OrgId) =>
+			CH.compile(CH.spanEventsByTraceQuery({ traceId: payload.traceId }), {
+				orgId,
+				startTime: payload.startTime,
+				endTime: payload.endTime,
+			}),
+	}),
+
 	podInfraTimeseries: defineQuery({
 		id: "podInfraTimeseries",
 		profile: "aggregation",

apps/web/src/api/warehouse/traces.ts

@@ -15,7 +15,10 @@ import {
 	SpanDetailRequest,
 	SpanHierarchyRequest,
 	SpanName,
+	TraceSpanEventsRequest,
 } from "@maple/domain/http"
+import type { SpanEvent } from "@maple/query-engine"
+import type { SpanEventsBySpanId } from "@maple/ui/lib/span-event-markers"
 import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client"
 import { computeTraceTimeWindow } from "@/lib/trace-time-window"
 import {
@@ -578,6 +581,7 @@ export type GetSpanDetailInput = Schema.Schema.Type<typeof GetSpanDetailInputSch
 export interface SpanDetailResult {
 	spanAttributes: Record<string, string>
 	resourceAttributes: Record<string, string>
+	events: ReadonlyArray<SpanEvent>
 }
 
 export function getSpanDetail({ data }: { data: GetSpanDetailInput }) {
@@ -612,6 +616,7 @@ const getSpanDetailEffect = Effect.fn("QueryEngine.getSpanDetail")(function* ({
 	return {
 		spanAttributes: parseAttributes(result.data?.spanAttributes),
 		resourceAttributes: parseAttributes(result.data?.resourceAttributes),
+		events: result.data?.events ?? [],
 	} satisfies SpanDetailResult
 })
 
@@ -757,6 +762,52 @@ const getTracesFacetsEffect = Effect.fn("QueryEngine.getTracesFacets")(function*
 	}
 })
 
+// Trace span events — event times and names for every span, drawn as markers on
+// the timeline bars. A separate request so the hierarchy never waits on `traces`.
+
+/** Slack around the trace's own span window for sub-second rounding and late events. */
+const TRACE_EVENTS_SLACK_MS = 1_000
+
+const GetTraceSpanEventsInputSchema = Schema.Struct({
+	traceId: TraceId,
+	/** Earliest span start (`SpanHierarchyResponse.traceStartTime`). */
+	traceStartTime: Schema.String,
+	totalDurationMs: Schema.Number,
+})
+
+export type GetTraceSpanEventsInput = Schema.Schema.Type<typeof GetTraceSpanEventsInputSchema>
+
+export function getTraceSpanEvents({ data }: { data: GetTraceSpanEventsInput }) {
+	return getTraceSpanEventsEffect({ data })
+}
+
+const getTraceSpanEventsEffect = Effect.fn("QueryEngine.getTraceSpanEvents")(function* ({
+	data,
+}: {
+	data: GetTraceSpanEventsInput
+}) {
+	const input = yield* decodeInput(GetTraceSpanEventsInputSchema, data, "getTraceSpanEvents")
+	yield* Effect.annotateCurrentSpan("traceId", input.traceId)
+
+	const startMs = Date.parse(input.traceStartTime)
+	if (Number.isNaN(startMs)) return new Map() satisfies SpanEventsBySpanId
+
+	const result = yield* runWarehouseQuery("traceSpanEvents", () =>
+		Effect.gen(function* () {
+			const client = yield* MapleInternalAtomClient
+			return yield* client.queryEngine.traceSpanEvents({
+				payload: new TraceSpanEventsRequest({
+					traceId: input.traceId,
+					startTime: formatWarehouseDateTime(startMs - TRACE_EVENTS_SLACK_MS),
+					endTime: formatWarehouseDateTime(startMs + input.totalDurationMs + TRACE_EVENTS_SLACK_MS),
+				}),
+			})
+		}),
+	)
+
+	return new Map(result.data.map((row) => [row.spanId, row.events])) satisfies SpanEventsBySpanId
+})
+
 // Single facet list for dashboard variables: compiles only the requested UNION
 // branch server-side and skips the duration-stats companion query, instead of
 // the full multi-facet scan `getTracesFacets` runs for the traces sidebar.

apps/web/src/components/traces/span-detail-panel.tsx

@@ -35,6 +35,7 @@ import {
 	ResourceAttributesSection,
 } from "@/components/attributes"
 import { TraceAttributeFilterProvider } from "./trace-attribute-filter-provider"
+import { SpanEventsSection } from "./span-events-section"
 import { useTimezonePreference } from "@/hooks/use-timezone-preference"
 import { formatTimestampInTimezone } from "@/lib/timezone-format"
 import { HttpSpanLabel } from "@maple/ui/components/traces/http-span-label"
@@ -473,6 +474,7 @@ export function SpanDetailPanel({
 									))
 									.onSuccess((detail) => (
 										<>
+											<SpanEventsSection events={detail.events} />
 											<TraceAttributeFilterProvider scope="span">
 												<AttributesSection
 													attributes={detail.spanAttributes}

apps/web/src/components/traces/span-events-section.tsx

@@ -0,0 +1,96 @@
+import type { SpanEvent } from "@maple/query-engine"
+import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@maple/ui/components/ui/collapsible"
+import { CopyButton } from "@maple/ui/components/ui/copy-button"
+import { KeyValue, KeyValueList } from "@maple/ui/components/ui/key-value"
+import { ChevronRightIcon, CircleXmarkIcon } from "@/components/icons"
+import { describeSpanEvent, formatEventOffset } from "@/lib/traces/span-events"
+
+function EventAttributes({ attributes }: { attributes: Readonly<Record<string, string>> }) {
+	const entries = Object.entries(attributes)
+	if (entries.length === 0) return null
+	return (
+		<KeyValueList className="gap-0.5 text-2xs">
+			{entries.map(([key, value]) => (
+				<KeyValue key={key} mono wrap label={key} copyValue={value}>
+					{value}
+				</KeyValue>
+			))}
+		</KeyValueList>
+	)
+}
+
+function SpanEventRow({ event }: { event: SpanEvent }) {
+	const described = describeSpanEvent(event)
+	const offset = (
+		<span
+			className="w-20 shrink-0 font-mono text-3xs tabular-nums text-muted-foreground"
+			title={event.timestamp}
+		>
+			{formatEventOffset(event.offsetMs)}
+		</span>
+	)
+
+	if (described.kind === "event") {
+		return (
+			<li className="space-y-1 p-2">
+				<div className="flex min-w-0 items-baseline gap-2">
+					{offset}
+					<span className="min-w-0 truncate font-mono text-xs">{event.name || "(unnamed)"}</span>
+				</div>
+				<EventAttributes attributes={described.attributes} />
+			</li>
+		)
+	}
+
+	return (
+		<li className="space-y-1 p-2">
+			<div className="flex min-w-0 items-baseline gap-2">
+				{offset}
+				<span className="flex min-w-0 items-center gap-1 font-mono text-xs text-severity-error">
+					<CircleXmarkIcon size={12} className="shrink-0 self-center" />
+					<span className="truncate">{described.type ?? "exception"}</span>
+				</span>
+			</div>
+			{described.message && (
+				<p className="font-mono text-xs whitespace-pre-wrap [overflow-wrap:anywhere]">
+					{described.message}
+				</p>
+			)}
+			{described.stacktrace && (
+				<Collapsible>
+					<div className="flex items-center gap-1">
+						<CollapsibleTrigger className="group flex items-center gap-1 text-3xs text-muted-foreground transition-colors hover:text-foreground">
+							<ChevronRightIcon
+								size={10}
+								className="transition-transform group-data-[panel-open]:rotate-90"
+							/>
+							Stack trace
+						</CollapsibleTrigger>
+						<CopyButton value={described.stacktrace} label="stack trace" iconSize={10} />
+					</div>
+					<CollapsibleContent>
+						<pre className="mt-1 max-h-80 overflow-auto rounded-sm bg-muted/40 p-2 font-mono text-3xs leading-relaxed">
+							{described.stacktrace}
+						</pre>
+					</CollapsibleContent>
+				</Collapsible>
+			)}
+			<EventAttributes attributes={described.attributes} />
+		</li>
+	)
+}
+
+/** The span's OTel events in time order, with `exception` events expanded. */
+export function SpanEventsSection({ events }: { events: ReadonlyArray<SpanEvent> }) {
+	if (events.length === 0) return null
+	return (
+		<div className="space-y-1">
+			<h4 className="text-xs font-medium text-muted-foreground">Events ({events.length})</h4>
+			<ul className="divide-y rounded-md border">
+				{events.map((event, index) => (
+					<SpanEventRow key={`${event.timestamp}-${event.name}-${index}`} event={event} />
+				))}
+			</ul>
+		</div>
+	)
+}

apps/web/src/lab/registry.ts

@@ -87,7 +87,7 @@ export const LAB_ENTRIES: ReadonlyArray<LabEntry> = [
 		path: "/lab/timeline",
 		title: "Trace timeline",
 		description:
-			"The trace view tabs over a synthetic trace with a wide dynamic range — a 2s root down to sub-millisecond spans that only resolve at deep zoom.",
+			"The trace view tabs over a synthetic trace with a wide dynamic range — a 2s root down to sub-millisecond spans that only resolve at deep zoom, with span event markers and the events panel.",
 		kind: "lab",
 		session: "none",
 	},