From f8a0e635293c5c960d3072d892bb6aea778e4d31 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 11 Oct 2026 01:19:07 +0200 Subject: [PATCH 1/5] feat(infra): show pod restarts and last termination reason Adds podRestartsQuery over the cluster receiver's k8s.container.restarts (window restarts = max - min of the running total per container) plus the k8s.container.status.last_terminated_reason resource attribute, a podRestarts endpoint, an inline restart marker on the pod list, and a restarts tile and per-container table on the pod page. --- .../src/routes/internal/query-engine.http.ts | 17 +++ apps/web/src/api/warehouse/infra.ts | 28 ++++ .../infra/kubernetes/pod-restarts-panel.tsx | 128 ++++++++++++++++++ .../src/components/infra/pod-restarts.test.ts | 74 ++++++++++ apps/web/src/components/infra/pod-restarts.ts | 54 ++++++++ apps/web/src/components/infra/pod-table.tsx | 25 +++- .../services/atoms/warehouse-query-atoms.ts | 5 + .../routes/infra/kubernetes/pods/$podName.tsx | 20 ++- .../routes/infra/kubernetes/pods/index.tsx | 7 +- deploy/k8s-infra/values.yaml | 2 + packages/domain/src/http/query-engine.ts | 35 +++++ .../query-engine/src/benchmark/builders.ts | 28 ++++ packages/query-engine/src/ch/index.ts | 3 + .../query-engine/src/ch/queries/infra.test.ts | 36 +++++ packages/query-engine/src/ch/queries/infra.ts | 56 ++++++++ packages/query-engine/src/registry/queries.ts | 16 +++ 16 files changed, 527 insertions(+), 7 deletions(-) create mode 100644 apps/web/src/components/infra/kubernetes/pod-restarts-panel.tsx create mode 100644 apps/web/src/components/infra/pod-restarts.test.ts create mode 100644 apps/web/src/components/infra/pod-restarts.ts diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 5bd9585d76..8ac03f0d6b 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -54,6 +54,7 @@ import { ListPodsResponse, PodsSummaryResponse, PodDetailSummaryResponse, + PodRestartsResponse, PodInfraTimeseriesResponse, PodFacetsResponse, ListContainersResponse, @@ -1552,6 +1553,22 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) + .handle("podRestarts", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.podRestarts, tenant, payload) + return new PodRestartsResponse({ + data: rows.map((row) => ({ + podName: row.podName, + namespace: row.namespace, + containerName: row.containerName, + restarts: Number(row.restarts) || 0, + totalRestarts: Number(row.totalRestarts) || 0, + lastTerminatedReason: row.lastTerminatedReason, + })), + }) + }), + ) .handle("podInfraTimeseries", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/web/src/api/warehouse/infra.ts b/apps/web/src/api/warehouse/infra.ts index a329181dc9..755e2214dc 100644 --- a/apps/web/src/api/warehouse/infra.ts +++ b/apps/web/src/api/warehouse/infra.ts @@ -6,6 +6,7 @@ import { ListPodsRequest, PodsSummaryRequest, PodDetailSummaryRequest, + PodRestartsRequest, PodInfraTimeseriesRequest, PodFacetsRequest, ListNodesRequest, @@ -28,6 +29,7 @@ import { type ListPodsResponse, type PodsSummaryResponse, type PodDetailSummaryResponse, + type PodRestartsResponse, type PodInfraTimeseriesResponse, type PodFacetsResponse, type ListNodesResponse, @@ -354,6 +356,32 @@ export function podDetailSummary({ data }: { data: PodDetailSummaryInput }) { ) } +export interface PodRestartsInput { + startTime: string + endTime: string + podNames?: ReadonlyArray + namespace?: string + limit?: number +} + +export function podRestarts({ data }: { data: PodRestartsInput }) { + return runWarehouseQuery("podRestarts", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + const response: PodRestartsResponse = yield* client.queryEngine.podRestarts({ + payload: new PodRestartsRequest({ + startTime: data.startTime, + endTime: data.endTime, + podNames: data.podNames, + namespace: data.namespace, + limit: data.limit, + }), + }) + return response + }), + ) +} + export type PodInfraMetric = "cpu_usage" | "cpu_limit" | "cpu_request" | "memory_limit" | "memory_request" export interface PodInfraTimeseriesInput { diff --git a/apps/web/src/components/infra/kubernetes/pod-restarts-panel.tsx b/apps/web/src/components/infra/kubernetes/pod-restarts-panel.tsx new file mode 100644 index 0000000000..739b304758 --- /dev/null +++ b/apps/web/src/components/infra/kubernetes/pod-restarts-panel.tsx @@ -0,0 +1,128 @@ +import type { ComponentProps } from "react" +import { EMPTY_VALUE } from "@maple/ui/lib/format" +import { TONE_TEXT } from "@maple/ui/lib/tone" +import { cn } from "@maple/ui/lib/utils" + +import { ColumnHead, DataTable } from "@/components/common/data-table" +import { SectionHeading } from "@/components/common/section-heading" +import { StatRailItem, StatRailItemSkeleton } from "@/components/common/stat-rail" +import { PodTable } from "@/components/infra/pod-table" +import { restartTone, summarizePodRestarts, type PodRestartRow } from "@/components/infra/pod-restarts" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { podRestartsResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" + +/** A page of 50 pods with a few containers each, with headroom for sidecars. */ +const PAGE_RESTARTS_LIMIT = 500 + +interface PodWindow { + podName: string + namespace?: string + startTime: string + endTime: string +} + +const podRestartsAtom = ({ podName, namespace, startTime, endTime }: PodWindow) => + podRestartsResultAtom({ data: { podNames: [podName], namespace, startTime, endTime } }) + +/** + * The pod table with each row's restarts beside its name. Restarts come from + * the cluster collector, not the kubelet, so they load as a second query and + * the table renders without them until it lands. Render only with a non-empty page. + */ +export function PodTableWithRestarts({ + startTime, + endTime, + ...props +}: Omit, "restarts"> & { startTime: string; endTime: string }) { + const result = useAtomValue( + podRestartsResultAtom({ + data: { + startTime, + endTime, + podNames: props.pods.map((pod) => pod.podName), + limit: PAGE_RESTARTS_LIMIT, + }, + }), + ) + const restarts = Result.builder(result) + .onSuccess((response) => summarizePodRestarts(response.data)) + .orElse(() => undefined) + return +} + +/** The restarts tile for the pod page's stat rail. */ +export function PodRestartsStat(props: PodWindow) { + const result = useAtomValue(podRestartsAtom(props)) + return Result.builder(result) + .onInitial(() => ) + .onError(() => ) + .onSuccess((response) => { + // No rows means the cluster collector isn't reporting this pod, not zero restarts. + if (response.data.length === 0) { + return + } + const summary = summarizePodRestarts(response.data).values().next().value + const restarts = summary?.restarts ?? 0 + const reason = summary?.lastTerminatedReason ?? "" + return ( + + ) + }) + .render() +} + +/** Per-container breakdown, shown once any container has restarted or terminated. */ +export function PodContainerRestarts(props: PodWindow) { + const result = useAtomValue(podRestartsAtom(props)) + const rows = Result.builder(result) + .onSuccess((response) => response.data) + .orElse(() => []) + if (!rows.some((row) => row.totalRestarts > 0 || row.lastTerminatedReason !== "")) return null + return ( +
+ + + + + + + + + {rows.map((row) => ( + + ))} + +
+ ) +} + +function ContainerRow({ row }: { row: PodRestartRow }) { + return ( +
+ + {row.containerName || EMPTY_VALUE} + + + {row.restarts} + + + {row.totalRestarts} + + + {row.lastTerminatedReason || EMPTY_VALUE} + +
+ ) +} diff --git a/apps/web/src/components/infra/pod-restarts.test.ts b/apps/web/src/components/infra/pod-restarts.test.ts new file mode 100644 index 0000000000..e80e419853 --- /dev/null +++ b/apps/web/src/components/infra/pod-restarts.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest" +import { formatRestarts, restartTone, summarizePodRestarts, type PodRestartRow } from "./pod-restarts" + +const row = (overrides: Partial): PodRestartRow => ({ + podName: "api-1", + namespace: "backend", + containerName: "app", + restarts: 0, + totalRestarts: 0, + lastTerminatedReason: "", + ...overrides, +}) + +describe("summarizePodRestarts", () => { + it("sums restarts across a pod's containers", () => { + const summary = summarizePodRestarts([ + row({ containerName: "app", restarts: 2, totalRestarts: 5 }), + row({ containerName: "sidecar", restarts: 1, totalRestarts: 1 }), + ]) + expect(summary.get("backend/api-1")).toEqual({ + restarts: 3, + totalRestarts: 6, + lastTerminatedReason: "", + }) + }) + + it("takes the reason from the container that restarted most in the window", () => { + const summary = summarizePodRestarts([ + row({ containerName: "sidecar", restarts: 1, totalRestarts: 9, lastTerminatedReason: "Error" }), + row({ containerName: "app", restarts: 4, totalRestarts: 4, lastTerminatedReason: "OOMKilled" }), + ]) + expect(summary.get("backend/api-1")?.lastTerminatedReason).toBe("OOMKilled") + }) + + it("falls back to any container with a reason when the top one has none", () => { + const summary = summarizePodRestarts([ + row({ containerName: "app", restarts: 3, lastTerminatedReason: "" }), + row({ containerName: "init", restarts: 0, totalRestarts: 1, lastTerminatedReason: "Completed" }), + ]) + expect(summary.get("backend/api-1")?.lastTerminatedReason).toBe("Completed") + }) + + it("keeps same-named pods in different namespaces apart", () => { + const summary = summarizePodRestarts([ + row({ namespace: "backend", restarts: 1 }), + row({ namespace: "staging", restarts: 7 }), + ]) + expect(summary.get("backend/api-1")?.restarts).toBe(1) + expect(summary.get("staging/api-1")?.restarts).toBe(7) + }) + + it("returns an empty map for no rows", () => { + expect(summarizePodRestarts([]).size).toBe(0) + }) +}) + +describe("restartTone", () => { + it("is neutral without restarts in the window, even with an old OOM kill", () => { + expect(restartTone(0, "OOMKilled")).toBe("neutral") + }) + + it("reads an OOM kill as critical and other restarts as a warning", () => { + expect(restartTone(2, "OOMKilled")).toBe("crit") + expect(restartTone(2, "Error")).toBe("warn") + expect(restartTone(1, "")).toBe("warn") + }) +}) + +describe("formatRestarts", () => { + it("pluralizes", () => { + expect(formatRestarts(1)).toBe("1 restart") + expect(formatRestarts(3)).toBe("3 restarts") + }) +}) diff --git a/apps/web/src/components/infra/pod-restarts.ts b/apps/web/src/components/infra/pod-restarts.ts new file mode 100644 index 0000000000..4bce4150ec --- /dev/null +++ b/apps/web/src/components/infra/pod-restarts.ts @@ -0,0 +1,54 @@ +import type { PodRestartsResponse } from "@maple/domain/http" +import type { Tone } from "@maple/ui/lib/tone" + +export type PodRestartRow = PodRestartsResponse["data"][number] + +export interface PodRestartSummary { + /** Restarts inside the window, summed over the pod's containers. */ + readonly restarts: number + /** Running totals summed over the pod's containers. */ + readonly totalRestarts: number + /** The reason from the container that restarted most, or any container that has one. */ + readonly lastTerminatedReason: string +} + +/** Row identity. A pod name repeats across namespaces; the pair doesn't. */ +export const podKey = (pod: { namespace: string; podName: string }) => `${pod.namespace}/${pod.podName}` + +const byMostRestarts = (a: PodRestartRow, b: PodRestartRow) => + b.restarts - a.restarts || b.totalRestarts - a.totalRestarts + +/** Folds per-container rows into one summary per pod, keyed by `podKey`. */ +export function summarizePodRestarts( + rows: ReadonlyArray, +): ReadonlyMap { + const byPod = new Map>() + for (const row of rows) { + const key = podKey(row) + const containers = byPod.get(key) + if (containers) containers.push(row) + else byPod.set(key, [row]) + } + return new Map( + [...byPod].map(([key, containers]) => { + const ranked = [...containers].sort(byMostRestarts) + return [ + key, + { + restarts: ranked.reduce((sum, c) => sum + c.restarts, 0), + totalRestarts: ranked.reduce((sum, c) => sum + c.totalRestarts, 0), + lastTerminatedReason: + ranked.find((c) => c.lastTerminatedReason !== "")?.lastTerminatedReason ?? "", + }, + ] + }), + ) +} + +/** An OOM kill is the one reason that says "raise the limit", so it reads as critical. */ +export function restartTone(restarts: number, reason: string): Tone { + if (restarts <= 0) return "neutral" + return reason === "OOMKilled" ? "crit" : "warn" +} + +export const formatRestarts = (n: number) => `${n} ${n === 1 ? "restart" : "restarts"}` diff --git a/apps/web/src/components/infra/pod-table.tsx b/apps/web/src/components/infra/pod-table.tsx index bb70d4b680..0bdb8a85fe 100644 --- a/apps/web/src/components/infra/pod-table.tsx +++ b/apps/web/src/components/infra/pod-table.tsx @@ -17,11 +17,11 @@ import { formatWholePercent } from "./format" import { MetaLine } from "./primitives/meta-line" import { RelativeTime } from "@/components/common/relative-time" import { TONE_TEXT } from "@maple/ui/lib/tone" +import { formatRestarts, podKey, restartTone, type PodRestartSummary } from "./pod-restarts" export type PodRow = ListPodsResponse["data"][number] -/** Row identity. A pod name repeats across namespaces; the pair doesn't. */ -export const podKey = (pod: Pick) => `${pod.namespace}/${pod.podName}` +export { podKey } interface PodTableProps { pods: ReadonlyArray @@ -48,6 +48,8 @@ interface PodTableProps { timeSearch?: TimeRangeSearch waiting?: boolean referenceTime?: string + /** Per-pod restarts by `podKey`, fetched for the page beside the list. Omitted, no marker. */ + restarts?: ReadonlyMap } const isPlainClick = (event: MouseEvent) => @@ -74,6 +76,23 @@ function metaTitle(pod: PodRow, workload: { kind: string; name: string } | null) .join(" · ") } +/** Inline beside the name, like the quiet status badge: absent unless the pod restarted in the window. */ +function RestartMarker({ summary }: { summary: PodRestartSummary | undefined }) { + if (!summary || summary.restarts <= 0) return null + const label = [formatRestarts(summary.restarts), summary.lastTerminatedReason].filter(Boolean).join(" · ") + return ( + + {label} + + ) +} + /** Cores read at very different magnitudes across a fleet; keep them comparable. */ const formatCores = (cores: number) => { if (!Number.isFinite(cores) || cores === 0) return "0" @@ -159,6 +178,7 @@ export function PodTable({ timeSearch, waiting, referenceTime, + restarts, }: PodTableProps) { return ( @@ -198,6 +218,7 @@ export function PodTable({ lastSeen={pod.lastSeen} referenceTime={referenceTime} /> + {podName}} - description="Pod metrics from the kubelet stats receiver." + description="Pod metrics from the kubelet stats receiver; restarts from the cluster receiver." meta={ <> {namespace && ns {namespace}} @@ -119,7 +120,7 @@ function PodDetailPage() { /> {Result.isInitial(summaryResult) ? ( - + ) : Result.isFailure(summaryResult) ? ( ) : summary ? ( - + + ) : ( @@ -171,6 +178,13 @@ function PodDetailPage() { bucketSeconds={bucketSeconds} /> + + ) diff --git a/apps/web/src/routes/infra/kubernetes/pods/index.tsx b/apps/web/src/routes/infra/kubernetes/pods/index.tsx index d15dd1177a..916827b7cd 100644 --- a/apps/web/src/routes/infra/kubernetes/pods/index.tsx +++ b/apps/web/src/routes/infra/kubernetes/pods/index.tsx @@ -19,7 +19,8 @@ import { InfraSetupEmpty } from "@/components/infra/infra-empty-state" import { KubernetesShell } from "@/components/infra/kubernetes/kubernetes-shell" import { PodPeekSheet } from "@/components/infra/kubernetes/pod-peek-sheet" import { PodsFilterSidebarView, type PodFilters } from "@/components/infra/k8s-filter-sidebar" -import { PodTable, PodTableLoading, podKey, type PodRow } from "@/components/infra/pod-table" +import { PodTableLoading, podKey, type PodRow } from "@/components/infra/pod-table" +import { PodTableWithRestarts } from "@/components/infra/kubernetes/pod-restarts-panel" import { FleetBand, FleetBandLoading, type FleetBandCell } from "@/components/infra/primitives/fleet-band" import { SearchToolbar, countLabel } from "@/components/common/search-toolbar" import { podFilterChips } from "@/lib/infra/pod-filter-chips" @@ -396,8 +397,10 @@ function PodsPage() { clearLabel="Clear all filters" /> ) : ( - ("PodRestartsRequest")({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + podNames: Schema.optional(StringArray), + namespace: Schema.optional(Schema.String), + limit: Schema.optional(Schema.Number), +}) {} + +export class PodRestartsResponse extends Schema.Class("PodRestartsResponse")({ + data: Schema.Array( + Schema.Struct({ + podName: Schema.String, + namespace: Schema.String, + containerName: Schema.String, + /** Restarts inside the window (max - min of the running total). */ + restarts: Schema.Number, + /** The running total at its highest in the window. */ + totalRestarts: Schema.Number, + /** Why the container last terminated (OOMKilled, Error, ...); empty when it never has. */ + lastTerminatedReason: Schema.String, + }), + ), +}) {} + export class PodInfraTimeseriesRequest extends Schema.Class( "PodInfraTimeseriesRequest", )({ @@ -2838,6 +2866,13 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) + .add( + HttpApiEndpoint.post("podRestarts", "/pod-restarts", { + payload: PodRestartsRequest, + success: PodRestartsResponse, + error: queryEngineEndpointErrors, + }), + ) .add( HttpApiEndpoint.post("podInfraTimeseries", "/pod-infra-timeseries", { payload: PodInfraTimeseriesRequest, diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index e13885d509..eded0c7856 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -1254,6 +1254,34 @@ export const builderFixtures: ReadonlyArray = [ { ...window, bucketSeconds: 300 }, ), }, + { + module: "infra", + name: "podRestartsQuery", + label: "default", + compile: () => + CH.compileUnsafe( + CH.podRestartsQuery({ + workloadKind: "deployment", + workloadName: "api", + namespace: "backend", + }), + window, + ), + }, + { + // The pod list's shape: one page of pod names, every container on them. + module: "infra", + name: "podRestartsQuery", + label: "podNames", + compile: () => + CH.compileUnsafe( + CH.podRestartsQuery({ + podNames: ["api-7d9f8b6c5-x2n4k", "worker-5c8d7f9b4-q7m2p"], + limit: 500, + }), + window, + ), + }, { module: "infra", name: "podGaugeTimeseriesQuery", diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 7ece54c1ab..838d3b3c21 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -593,6 +593,7 @@ export { listPodsQuery, listPodsSummaryQuery, podDetailSummaryQuery, + podRestartsQuery, podGaugeTimeseriesQuery, podFacetsQuery, listNodesQuery, @@ -618,6 +619,8 @@ export { type SortDirection, type PodDetailSummaryOpts, type PodDetailSummaryOutput, + type PodRestartsOpts, + type PodRestartsOutput, type PodGaugeTimeseriesOpts, type PodFacetsOutput, type ListNodesOpts, diff --git a/packages/query-engine/src/ch/queries/infra.test.ts b/packages/query-engine/src/ch/queries/infra.test.ts index ead6df77dd..3b7670b94d 100644 --- a/packages/query-engine/src/ch/queries/infra.test.ts +++ b/packages/query-engine/src/ch/queries/infra.test.ts @@ -10,6 +10,7 @@ import { listPodsQuery, listPodsSummaryQuery, podDetailSummaryQuery, + podRestartsQuery, podGaugeTimeseriesQuery, podFacetsQuery, listNodesQuery, @@ -299,6 +300,41 @@ describe("podDetailSummaryQuery", () => { }) }) +describe("podRestartsQuery", () => { + it("reads the running total per container and scopes to the org and window", () => { + const { sql } = compileUnsafe(podRestartsQuery({}), baseParams) + expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("MetricName = 'k8s.container.restarts'") + expect(sql).toContain("max(metrics_gauge.Value) - min(metrics_gauge.Value)") + expect(sql).toContain("ResourceAttributes['k8s.container.status.last_terminated_reason']") + expect(sql).toMatch(/GROUP BY podName, namespace, containerName/) + expect(sql).not.toMatch(/__PARAM_\w+__/) + }) + + it("narrows to the page's pods, a namespace and a workload", () => { + const { sql } = compileUnsafe( + podRestartsQuery({ + podNames: ["api-1", "api-2"], + namespace: "backend", + workloadKind: "statefulset", + workloadName: "db", + limit: 500, + }), + baseParams, + ) + expect(sql).toContain("'api-1'") + expect(sql).toContain("'api-2'") + expect(sql).toContain("ResourceAttributes['k8s.namespace.name'] = 'backend'") + expect(sql).toContain("ResourceAttributes['k8s.statefulset.name'] = 'db'") + expect(sql).toContain("LIMIT 500") + }) + + it("skips the pod-name filter for an empty page rather than emitting IN ()", () => { + const { sql } = compileUnsafe(podRestartsQuery({ podNames: [] }), baseParams) + expect(sql).not.toMatch(/IN \(\s*\)/) + }) +}) + describe("podGaugeTimeseriesQuery", () => { it("buckets by toStartOfInterval and filters by metric name", () => { const { sql } = compileUnsafe( diff --git a/packages/query-engine/src/ch/queries/infra.ts b/packages/query-engine/src/ch/queries/infra.ts index 15c27ee3f6..469a545c44 100644 --- a/packages/query-engine/src/ch/queries/infra.ts +++ b/packages/query-engine/src/ch/queries/infra.ts @@ -675,6 +675,62 @@ export function podDetailSummaryQuery(opts: PodDetailSummaryOpts) { .format("JSON") } +// Pod container restarts, from the k8s_cluster receiver's `k8s.container.restarts` +// (a running total per container). The window's restarts are max - min per container; +// a kubelet that prunes dead containers can reset the total, so this undercounts, never over. + +export interface PodRestartsOpts { + podNames?: ReadonlyArray + namespace?: string + workloadKind?: "deployment" | "statefulset" | "daemonset" + workloadName?: string + limit?: number +} + +export interface PodRestartsOutput { + readonly podName: string + readonly namespace: string + readonly containerName: string + readonly restarts: number + readonly totalRestarts: number + /** `k8s.container.status.last_terminated_reason` (OOMKilled, Error, ...); empty when never terminated. */ + readonly lastTerminatedReason: string +} + +export function podRestartsQuery(opts: PodRestartsOpts) { + return from(MetricsGauge) + .select(($) => ({ + podName: $.ResourceAttributes.get("k8s.pod.name"), + namespace: $.ResourceAttributes.get("k8s.namespace.name"), + containerName: $.ResourceAttributes.get("k8s.container.name"), + restarts: CH.ifNotFinite(CH.max_($.Value).sub(CH.min_($.Value)), 0), + totalRestarts: CH.ifNotFinite(CH.max_($.Value), 0), + // The attribute rides on the resource, so the newest sample carries the newest reason. + lastTerminatedReason: CH.argMax( + $.ResourceAttributes.get("k8s.container.status.last_terminated_reason"), + $.TimeUnix, + ), + })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.TimeUnix.gte(param.dateTime("startTime")), + $.TimeUnix.lte(param.dateTime("endTime")), + $.MetricName.eq("k8s.container.restarts"), + $.ResourceAttributes.get("k8s.pod.name").neq(""), + opts.podNames?.length + ? CH.inList($.ResourceAttributes.get("k8s.pod.name"), opts.podNames) + : undefined, + CH.when(opts.namespace, (v: string) => $.ResourceAttributes.get("k8s.namespace.name").eq(v)), + CH.when(opts.workloadName, (v: string) => + $.ResourceAttributes.get(workloadAttrKey(opts.workloadKind ?? "deployment")).eq(v), + ), + ]) + .groupBy("podName", "namespace", "containerName") + .orderBy(["restarts", "desc"], ["totalRestarts", "desc"]) + .limit(opts.limit ?? 50) + .format("JSON") +} + // Pod time-series — gauge metric for one pod, optionally broken down by an // attribute key (e.g. container name, when present). diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index 786275afd6..31946e4c57 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -29,6 +29,7 @@ import type { MetricsSummaryRequest, NodeDetailSummaryRequest, PodDetailSummaryRequest, + PodRestartsRequest, PodsSummaryRequest, ServiceApdexRequest, ServiceDbEdgesForServiceRequest, @@ -447,6 +448,21 @@ export const podDetailSummary = defineQuery({ }), }) +export const podRestarts = defineQuery({ + id: "podRestarts", + profile: "aggregation", + cache: timeRangeCache, + compile: (payload: PodRestartsRequest, orgId: OrgId) => + CH.compile( + CH.podRestartsQuery({ + podNames: payload.podNames, + namespace: payload.namespace, + limit: payload.limit, + }), + { orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) + export const listNodes = defineQuery({ id: "listNodes", profile: "list", From 41f96a4f0c437400e24a632e94fe049667a1a778 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 12 Oct 2026 02:27:44 +0200 Subject: [PATCH 2/5] test(query-engine): record podRestarts in the SQL catalog baseline --- .../src/__sql_baseline__/catalog.sql | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 251b75e0df..37f6e200d9 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -1371,6 +1371,47 @@ SELECT ORDER BY bucket ASC FORMAT JSON +-- builder:infra:podRestartsQuery:default [12aafde3] +SELECT + metrics_gauge.ResourceAttributes['k8s.pod.name'] AS podName, + metrics_gauge.ResourceAttributes['k8s.namespace.name'] AS namespace, + metrics_gauge.ResourceAttributes['k8s.container.name'] AS containerName, + ifNotFinite(max(metrics_gauge.Value) - min(metrics_gauge.Value), 0) AS restarts, + ifNotFinite(max(metrics_gauge.Value), 0) AS totalRestarts, + argMax(metrics_gauge.ResourceAttributes['k8s.container.status.last_terminated_reason'], metrics_gauge.TimeUnix) AS lastTerminatedReason + FROM metrics_gauge + WHERE metrics_gauge.OrgId = 'org_sql_catalog' + AND metrics_gauge.TimeUnix >= '2026-01-01 10:30:00' + AND metrics_gauge.TimeUnix <= '2026-01-03 14:15:00' + AND metrics_gauge.MetricName = 'k8s.container.restarts' + AND metrics_gauge.ResourceAttributes['k8s.pod.name'] != '' + AND metrics_gauge.ResourceAttributes['k8s.namespace.name'] = 'backend' + AND metrics_gauge.ResourceAttributes['k8s.deployment.name'] = 'api' + GROUP BY podName, namespace, containerName + ORDER BY restarts DESC, totalRestarts DESC + LIMIT 50 + FORMAT JSON + +-- builder:infra:podRestartsQuery:podNames [ede2928b] +SELECT + metrics_gauge.ResourceAttributes['k8s.pod.name'] AS podName, + metrics_gauge.ResourceAttributes['k8s.namespace.name'] AS namespace, + metrics_gauge.ResourceAttributes['k8s.container.name'] AS containerName, + ifNotFinite(max(metrics_gauge.Value) - min(metrics_gauge.Value), 0) AS restarts, + ifNotFinite(max(metrics_gauge.Value), 0) AS totalRestarts, + argMax(metrics_gauge.ResourceAttributes['k8s.container.status.last_terminated_reason'], metrics_gauge.TimeUnix) AS lastTerminatedReason + FROM metrics_gauge + WHERE metrics_gauge.OrgId = 'org_sql_catalog' + AND metrics_gauge.TimeUnix >= '2026-01-01 10:30:00' + AND metrics_gauge.TimeUnix <= '2026-01-03 14:15:00' + AND metrics_gauge.MetricName = 'k8s.container.restarts' + AND metrics_gauge.ResourceAttributes['k8s.pod.name'] != '' + AND metrics_gauge.ResourceAttributes['k8s.pod.name'] IN ('api-7d9f8b6c5-x2n4k', 'worker-5c8d7f9b4-q7m2p') + GROUP BY podName, namespace, containerName + ORDER BY restarts DESC, totalRestarts DESC + LIMIT 500 + FORMAT JSON + -- builder:infra:workloadFacetsQuery:default [39bce5d0] SELECT metrics_gauge.ResourceAttributes['k8s.deployment.name'] AS name, From 46015d9e1b44913b9a85afc6c38b3ed31d9307d3 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 12 Oct 2026 02:31:50 +0200 Subject: [PATCH 3/5] fix(query-engine): count pod restarts as last minus first so a counter reset cannot overcount --- .../query-engine/src/__sql_baseline__/catalog.sql | 12 ++++++------ packages/query-engine/src/ch/queries/infra.test.ts | 4 +++- packages/query-engine/src/ch/queries/infra.ts | 9 ++++++--- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 52ddbb69ac..821a33fa3f 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -1459,13 +1459,13 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- builder:infra:podRestartsQuery:default [12aafde3] +-- builder:infra:podRestartsQuery:default [b489a972] SELECT metrics_gauge.ResourceAttributes['k8s.pod.name'] AS podName, metrics_gauge.ResourceAttributes['k8s.namespace.name'] AS namespace, metrics_gauge.ResourceAttributes['k8s.container.name'] AS containerName, - ifNotFinite(max(metrics_gauge.Value) - min(metrics_gauge.Value), 0) AS restarts, - ifNotFinite(max(metrics_gauge.Value), 0) AS totalRestarts, + ifNotFinite(greatest(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix) - argMin(metrics_gauge.Value, metrics_gauge.TimeUnix), 0), 0) AS restarts, + ifNotFinite(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix), 0) AS totalRestarts, argMax(metrics_gauge.ResourceAttributes['k8s.container.status.last_terminated_reason'], metrics_gauge.TimeUnix) AS lastTerminatedReason FROM metrics_gauge WHERE metrics_gauge.OrgId = 'org_sql_catalog' @@ -1480,13 +1480,13 @@ SELECT LIMIT 50 FORMAT JSON --- builder:infra:podRestartsQuery:podNames [ede2928b] +-- builder:infra:podRestartsQuery:podNames [4976b882] SELECT metrics_gauge.ResourceAttributes['k8s.pod.name'] AS podName, metrics_gauge.ResourceAttributes['k8s.namespace.name'] AS namespace, metrics_gauge.ResourceAttributes['k8s.container.name'] AS containerName, - ifNotFinite(max(metrics_gauge.Value) - min(metrics_gauge.Value), 0) AS restarts, - ifNotFinite(max(metrics_gauge.Value), 0) AS totalRestarts, + ifNotFinite(greatest(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix) - argMin(metrics_gauge.Value, metrics_gauge.TimeUnix), 0), 0) AS restarts, + ifNotFinite(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix), 0) AS totalRestarts, argMax(metrics_gauge.ResourceAttributes['k8s.container.status.last_terminated_reason'], metrics_gauge.TimeUnix) AS lastTerminatedReason FROM metrics_gauge WHERE metrics_gauge.OrgId = 'org_sql_catalog' diff --git a/packages/query-engine/src/ch/queries/infra.test.ts b/packages/query-engine/src/ch/queries/infra.test.ts index c852297b82..d054f95c47 100644 --- a/packages/query-engine/src/ch/queries/infra.test.ts +++ b/packages/query-engine/src/ch/queries/infra.test.ts @@ -342,7 +342,9 @@ describe("podRestartsQuery", () => { const { sql } = compileUnsafe(podRestartsQuery({}), baseParams) expect(sql).toContain("OrgId = 'org_1'") expect(sql).toContain("MetricName = 'k8s.container.restarts'") - expect(sql).toContain("max(metrics_gauge.Value) - min(metrics_gauge.Value)") + expect(sql).toContain( + "greatest(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix) - argMin(metrics_gauge.Value, metrics_gauge.TimeUnix), 0)", + ) expect(sql).toContain("ResourceAttributes['k8s.container.status.last_terminated_reason']") expect(sql).toMatch(/GROUP BY podName, namespace, containerName/) expect(sql).not.toMatch(/__PARAM_\w+__/) diff --git a/packages/query-engine/src/ch/queries/infra.ts b/packages/query-engine/src/ch/queries/infra.ts index 3509d24686..f395726096 100644 --- a/packages/query-engine/src/ch/queries/infra.ts +++ b/packages/query-engine/src/ch/queries/infra.ts @@ -736,7 +736,7 @@ export function podDetailSummaryQuery(opts: PodDetailSummaryOpts) { } // Pod container restarts, from the k8s_cluster receiver's `k8s.container.restarts` -// (a running total per container). The window's restarts are max - min per container; +// (a running total per container). The window's restarts are last - first, floored at 0: // a kubelet that prunes dead containers can reset the total, so this undercounts, never over. export interface PodRestartsOpts { @@ -763,8 +763,11 @@ export function podRestartsQuery(opts: PodRestartsOpts) { podName: $.ResourceAttributes.get("k8s.pod.name"), namespace: $.ResourceAttributes.get("k8s.namespace.name"), containerName: $.ResourceAttributes.get("k8s.container.name"), - restarts: CH.ifNotFinite(CH.max_($.Value).sub(CH.min_($.Value)), 0), - totalRestarts: CH.ifNotFinite(CH.max_($.Value), 0), + restarts: CH.ifNotFinite( + CH.greatest_(CH.argMax($.Value, $.TimeUnix).sub(CH.argMin($.Value, $.TimeUnix)), CH.lit(0)), + 0, + ), + totalRestarts: CH.ifNotFinite(CH.argMax($.Value, $.TimeUnix), 0), // The attribute rides on the resource, so the newest sample carries the newest reason. lastTerminatedReason: CH.argMax( $.ResourceAttributes.get("k8s.container.status.last_terminated_reason"), From a93a7eff74d7e42fbbfd98bb222f9e46b5c2a40c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 12 Oct 2026 02:36:44 +0200 Subject: [PATCH 4/5] chore: clear knip findings inherited from main (dead infra presets, OverviewGrain, MCP playground entries) --- apps/web/src/components/infra/constants.ts | 26 +------------------ knip.json | 3 ++- .../query-engine/src/ch/queries/services.ts | 5 ++-- 3 files changed, 5 insertions(+), 29 deletions(-) diff --git a/apps/web/src/components/infra/constants.ts b/apps/web/src/components/infra/constants.ts index 9d7f0ab2b1..6ef4a5fc55 100644 --- a/apps/web/src/components/infra/constants.ts +++ b/apps/web/src/components/infra/constants.ts @@ -1,31 +1,7 @@ import { toEpochMs } from "@maple/ui/lib/time-format" // Shared controls for the infra detail routes. Per-resource metric tabs/strips -// stay in their own routes (genuinely page-specific); only the time-range -// vocabulary is common. - -export const TIME_PRESETS = [ - { value: "15m", label: "Last 15 minutes" }, - { value: "1h", label: "Last hour" }, - { value: "6h", label: "Last 6 hours" }, - { value: "12h", label: "Last 12 hours" }, - { value: "24h", label: "Last 24 hours" }, - { value: "7d", label: "Last 7 days" }, -] as const - -const BUCKET_SECONDS: Record = { - "15m": 15, - "1h": 60, - "6h": 300, - "12h": 600, - "24h": 900, - "7d": 3600, -} satisfies Record - -/** Chart bucket width for a preset; falls back to 60s for anything unrecognized. */ -export function bucketSecondsFor(preset: string): number { - return BUCKET_SECONDS[preset] ?? 60 -} +// stay in their own routes (genuinely page-specific); only the bucket ladder is common. /** * Bucket widths a kubelet-sampled series can honestly be drawn at. The floor is diff --git a/knip.json b/knip.json index ea78464aad..9bb3a32bca 100644 --- a/knip.json +++ b/knip.json @@ -43,8 +43,9 @@ }, // Single-module Workers: the root alchemy.run.ts imports src/worker.ts, and // alchemy bundles it, so it is the app's one entry. + // The tool playground runs through its own vitest config from scripts/tool-playground.sh. "apps/ai": { - "entry": ["src/worker.ts"] + "entry": ["src/worker.ts", "vitest.playground.config.ts", "src/evals/playground/*.playground.ts"] }, "apps/alerting": { "entry": ["src/worker.ts"] diff --git a/packages/query-engine/src/ch/queries/services.ts b/packages/query-engine/src/ch/queries/services.ts index 9205cf368a..c69bd89669 100644 --- a/packages/query-engine/src/ch/queries/services.ts +++ b/packages/query-engine/src/ch/queries/services.ts @@ -75,10 +75,9 @@ export interface ServiceWindowFilters { * `bucketSeconds` is not a multiple of 3600 — an hour-floored row landing in a * sub-hour bucket puts a whole hour of traffic on the bucket containing `:00` * and leaves the rest of the hour reading zero. + * + * The hour grain always splices the hourly tier; only the minute grain can drop it. */ -export type OverviewGrain = "hour" | "minute" - -/** The hour grain always splices the hourly tier; only the minute grain can drop it. */ export type ServiceWindowTiers = | { readonly grain?: "hour"; readonly includeHourly?: true } | { From 4c6375bf91d95e40474422d8937779132d3e0991 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 12 Oct 2026 02:37:24 +0200 Subject: [PATCH 5/5] docs(domain): describe pod restarts as last minus first --- packages/domain/src/http/query-engine.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 9ea1e5da61..d76f93eda1 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -2136,9 +2136,9 @@ export class PodRestartsResponse extends Schema.Class("PodR podName: Schema.String, namespace: Schema.String, containerName: Schema.String, - /** Restarts inside the window (max - min of the running total). */ + /** Restarts inside the window (last - first of the running total, floored at 0). */ restarts: Schema.Number, - /** The running total at its highest in the window. */ + /** The running total at the window's newest sample. */ totalRestarts: Schema.Number, /** Why the container last terminated (OOMKilled, Error, ...); empty when it never has. */ lastTerminatedReason: Schema.String,