diff --git a/apps/ai/src/mcp/__evals__/infra-world.ts b/apps/ai/src/mcp/__evals__/infra-world.ts index b1021ae9bd..02603e40a7 100644 --- a/apps/ai/src/mcp/__evals__/infra-world.ts +++ b/apps/ai/src/mcp/__evals__/infra-world.ts @@ -597,6 +597,7 @@ export const infraFixtureRules = (): FixtureRule[] => [ containerName: p.owner[1], restarts: RESTARTS.get(p.podName) ?? 0, totalRestarts: (RESTARTS.get(p.podName) ?? 0) + 2, + lastTerminatedReason: RESTARTS.has(p.podName) ? "OOMKilled" : "", })), }, { diff --git a/apps/ai/src/mcp/tools/__tests__/infra-tools.test.ts b/apps/ai/src/mcp/tools/__tests__/infra-tools.test.ts index e8f82319f3..f1b92878e0 100644 --- a/apps/ai/src/mcp/tools/__tests__/infra-tools.test.ts +++ b/apps/ai/src/mcp/tools/__tests__/infra-tools.test.ts @@ -77,6 +77,7 @@ describe("list_infra", () => { containerName: "checkout", restarts: 1, totalRestarts: 3, + lastTerminatedReason: "", }, { namespace: "staging", @@ -84,6 +85,7 @@ describe("list_infra", () => { containerName: "checkout", restarts: 7, totalRestarts: 9, + lastTerminatedReason: "", }, ], }) diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 8875d2e84f..020cf8adf6 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -54,6 +54,7 @@ import { ListPodsResponse, PodsSummaryResponse, PodDetailSummaryResponse, + PodRestartsResponse, PodInfraTimeseriesResponse, PodFacetsResponse, ListContainersResponse, @@ -1562,6 +1563,22 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) + .handle("podRestarts", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.podRestarts, tenant, payload) + return new PodRestartsResponse({ + data: rows.map((row) => ({ + podName: row.podName, + namespace: row.namespace, + containerName: row.containerName, + restarts: Number(row.restarts) || 0, + totalRestarts: Number(row.totalRestarts) || 0, + lastTerminatedReason: row.lastTerminatedReason, + })), + }) + }), + ) .handle("podInfraTimeseries", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/web/src/api/warehouse/infra.ts b/apps/web/src/api/warehouse/infra.ts index 3aa6fae175..fe939f42c1 100644 --- a/apps/web/src/api/warehouse/infra.ts +++ b/apps/web/src/api/warehouse/infra.ts @@ -6,6 +6,7 @@ import { ListPodsRequest, PodsSummaryRequest, PodDetailSummaryRequest, + PodRestartsRequest, PodInfraTimeseriesRequest, PodFacetsRequest, ListNodesRequest, @@ -28,6 +29,7 @@ import { type ListPodsResponse, type PodsSummaryResponse, type PodDetailSummaryResponse, + type PodRestartsResponse, type PodInfraTimeseriesResponse, type PodFacetsResponse, type ListNodesResponse, @@ -354,6 +356,32 @@ export function podDetailSummary({ data }: { data: PodDetailSummaryInput }) { ) } +export interface PodRestartsInput { + startTime: string + endTime: string + podNames?: ReadonlyArray + namespace?: string + limit?: number +} + +export function podRestarts({ data }: { data: PodRestartsInput }) { + return runWarehouseQuery("podRestarts", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + const response: PodRestartsResponse = yield* client.queryEngine.podRestarts({ + payload: new PodRestartsRequest({ + startTime: data.startTime, + endTime: data.endTime, + podNames: data.podNames, + namespace: data.namespace, + limit: data.limit, + }), + }) + return response + }), + ) +} + export type PodInfraMetric = "cpu_usage" | "cpu_limit" | "cpu_request" | "memory_limit" | "memory_request" export interface PodInfraTimeseriesInput { diff --git a/apps/web/src/components/infra/kubernetes/pod-restarts-panel.tsx b/apps/web/src/components/infra/kubernetes/pod-restarts-panel.tsx new file mode 100644 index 0000000000..739b304758 --- /dev/null +++ b/apps/web/src/components/infra/kubernetes/pod-restarts-panel.tsx @@ -0,0 +1,128 @@ +import type { ComponentProps } from "react" +import { EMPTY_VALUE } from "@maple/ui/lib/format" +import { TONE_TEXT } from "@maple/ui/lib/tone" +import { cn } from "@maple/ui/lib/utils" + +import { ColumnHead, DataTable } from "@/components/common/data-table" +import { SectionHeading } from "@/components/common/section-heading" +import { StatRailItem, StatRailItemSkeleton } from "@/components/common/stat-rail" +import { PodTable } from "@/components/infra/pod-table" +import { restartTone, summarizePodRestarts, type PodRestartRow } from "@/components/infra/pod-restarts" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { podRestartsResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" + +/** A page of 50 pods with a few containers each, with headroom for sidecars. */ +const PAGE_RESTARTS_LIMIT = 500 + +interface PodWindow { + podName: string + namespace?: string + startTime: string + endTime: string +} + +const podRestartsAtom = ({ podName, namespace, startTime, endTime }: PodWindow) => + podRestartsResultAtom({ data: { podNames: [podName], namespace, startTime, endTime } }) + +/** + * The pod table with each row's restarts beside its name. Restarts come from + * the cluster collector, not the kubelet, so they load as a second query and + * the table renders without them until it lands. Render only with a non-empty page. + */ +export function PodTableWithRestarts({ + startTime, + endTime, + ...props +}: Omit, "restarts"> & { startTime: string; endTime: string }) { + const result = useAtomValue( + podRestartsResultAtom({ + data: { + startTime, + endTime, + podNames: props.pods.map((pod) => pod.podName), + limit: PAGE_RESTARTS_LIMIT, + }, + }), + ) + const restarts = Result.builder(result) + .onSuccess((response) => summarizePodRestarts(response.data)) + .orElse(() => undefined) + return +} + +/** The restarts tile for the pod page's stat rail. */ +export function PodRestartsStat(props: PodWindow) { + const result = useAtomValue(podRestartsAtom(props)) + return Result.builder(result) + .onInitial(() => ) + .onError(() => ) + .onSuccess((response) => { + // No rows means the cluster collector isn't reporting this pod, not zero restarts. + if (response.data.length === 0) { + return + } + const summary = summarizePodRestarts(response.data).values().next().value + const restarts = summary?.restarts ?? 0 + const reason = summary?.lastTerminatedReason ?? "" + return ( + + ) + }) + .render() +} + +/** Per-container breakdown, shown once any container has restarted or terminated. */ +export function PodContainerRestarts(props: PodWindow) { + const result = useAtomValue(podRestartsAtom(props)) + const rows = Result.builder(result) + .onSuccess((response) => response.data) + .orElse(() => []) + if (!rows.some((row) => row.totalRestarts > 0 || row.lastTerminatedReason !== "")) return null + return ( +
+ + + + + + + + + {rows.map((row) => ( + + ))} + +
+ ) +} + +function ContainerRow({ row }: { row: PodRestartRow }) { + return ( +
+ + {row.containerName || EMPTY_VALUE} + + + {row.restarts} + + + {row.totalRestarts} + + + {row.lastTerminatedReason || EMPTY_VALUE} + +
+ ) +} diff --git a/apps/web/src/components/infra/pod-restarts.test.ts b/apps/web/src/components/infra/pod-restarts.test.ts new file mode 100644 index 0000000000..e80e419853 --- /dev/null +++ b/apps/web/src/components/infra/pod-restarts.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest" +import { formatRestarts, restartTone, summarizePodRestarts, type PodRestartRow } from "./pod-restarts" + +const row = (overrides: Partial): PodRestartRow => ({ + podName: "api-1", + namespace: "backend", + containerName: "app", + restarts: 0, + totalRestarts: 0, + lastTerminatedReason: "", + ...overrides, +}) + +describe("summarizePodRestarts", () => { + it("sums restarts across a pod's containers", () => { + const summary = summarizePodRestarts([ + row({ containerName: "app", restarts: 2, totalRestarts: 5 }), + row({ containerName: "sidecar", restarts: 1, totalRestarts: 1 }), + ]) + expect(summary.get("backend/api-1")).toEqual({ + restarts: 3, + totalRestarts: 6, + lastTerminatedReason: "", + }) + }) + + it("takes the reason from the container that restarted most in the window", () => { + const summary = summarizePodRestarts([ + row({ containerName: "sidecar", restarts: 1, totalRestarts: 9, lastTerminatedReason: "Error" }), + row({ containerName: "app", restarts: 4, totalRestarts: 4, lastTerminatedReason: "OOMKilled" }), + ]) + expect(summary.get("backend/api-1")?.lastTerminatedReason).toBe("OOMKilled") + }) + + it("falls back to any container with a reason when the top one has none", () => { + const summary = summarizePodRestarts([ + row({ containerName: "app", restarts: 3, lastTerminatedReason: "" }), + row({ containerName: "init", restarts: 0, totalRestarts: 1, lastTerminatedReason: "Completed" }), + ]) + expect(summary.get("backend/api-1")?.lastTerminatedReason).toBe("Completed") + }) + + it("keeps same-named pods in different namespaces apart", () => { + const summary = summarizePodRestarts([ + row({ namespace: "backend", restarts: 1 }), + row({ namespace: "staging", restarts: 7 }), + ]) + expect(summary.get("backend/api-1")?.restarts).toBe(1) + expect(summary.get("staging/api-1")?.restarts).toBe(7) + }) + + it("returns an empty map for no rows", () => { + expect(summarizePodRestarts([]).size).toBe(0) + }) +}) + +describe("restartTone", () => { + it("is neutral without restarts in the window, even with an old OOM kill", () => { + expect(restartTone(0, "OOMKilled")).toBe("neutral") + }) + + it("reads an OOM kill as critical and other restarts as a warning", () => { + expect(restartTone(2, "OOMKilled")).toBe("crit") + expect(restartTone(2, "Error")).toBe("warn") + expect(restartTone(1, "")).toBe("warn") + }) +}) + +describe("formatRestarts", () => { + it("pluralizes", () => { + expect(formatRestarts(1)).toBe("1 restart") + expect(formatRestarts(3)).toBe("3 restarts") + }) +}) diff --git a/apps/web/src/components/infra/pod-restarts.ts b/apps/web/src/components/infra/pod-restarts.ts new file mode 100644 index 0000000000..4bce4150ec --- /dev/null +++ b/apps/web/src/components/infra/pod-restarts.ts @@ -0,0 +1,54 @@ +import type { PodRestartsResponse } from "@maple/domain/http" +import type { Tone } from "@maple/ui/lib/tone" + +export type PodRestartRow = PodRestartsResponse["data"][number] + +export interface PodRestartSummary { + /** Restarts inside the window, summed over the pod's containers. */ + readonly restarts: number + /** Running totals summed over the pod's containers. */ + readonly totalRestarts: number + /** The reason from the container that restarted most, or any container that has one. */ + readonly lastTerminatedReason: string +} + +/** Row identity. A pod name repeats across namespaces; the pair doesn't. */ +export const podKey = (pod: { namespace: string; podName: string }) => `${pod.namespace}/${pod.podName}` + +const byMostRestarts = (a: PodRestartRow, b: PodRestartRow) => + b.restarts - a.restarts || b.totalRestarts - a.totalRestarts + +/** Folds per-container rows into one summary per pod, keyed by `podKey`. */ +export function summarizePodRestarts( + rows: ReadonlyArray, +): ReadonlyMap { + const byPod = new Map>() + for (const row of rows) { + const key = podKey(row) + const containers = byPod.get(key) + if (containers) containers.push(row) + else byPod.set(key, [row]) + } + return new Map( + [...byPod].map(([key, containers]) => { + const ranked = [...containers].sort(byMostRestarts) + return [ + key, + { + restarts: ranked.reduce((sum, c) => sum + c.restarts, 0), + totalRestarts: ranked.reduce((sum, c) => sum + c.totalRestarts, 0), + lastTerminatedReason: + ranked.find((c) => c.lastTerminatedReason !== "")?.lastTerminatedReason ?? "", + }, + ] + }), + ) +} + +/** An OOM kill is the one reason that says "raise the limit", so it reads as critical. */ +export function restartTone(restarts: number, reason: string): Tone { + if (restarts <= 0) return "neutral" + return reason === "OOMKilled" ? "crit" : "warn" +} + +export const formatRestarts = (n: number) => `${n} ${n === 1 ? "restart" : "restarts"}` diff --git a/apps/web/src/components/infra/pod-table.tsx b/apps/web/src/components/infra/pod-table.tsx index b97cc4f0e6..e982c49db8 100644 --- a/apps/web/src/components/infra/pod-table.tsx +++ b/apps/web/src/components/infra/pod-table.tsx @@ -18,11 +18,11 @@ import { formatWholePercent } from "./format" import { MetaLine } from "./primitives/meta-line" import { RelativeTime } from "@/components/common/relative-time" import { TONE_TEXT } from "@maple/ui/lib/tone" +import { formatRestarts, podKey, restartTone, type PodRestartSummary } from "./pod-restarts" export type PodRow = ListPodsResponse["data"][number] -/** Row identity. A pod name repeats across namespaces; the pair doesn't. */ -export const podKey = (pod: Pick) => `${pod.namespace}/${pod.podName}` +export { podKey } interface PodTableProps { pods: ReadonlyArray @@ -49,6 +49,8 @@ interface PodTableProps { timeSearch?: TimeRangeSearch waiting?: boolean referenceTime?: string + /** Per-pod restarts by `podKey`, fetched for the page beside the list. Omitted, no marker. */ + restarts?: ReadonlyMap } const isPlainClick = (event: MouseEvent) => @@ -75,6 +77,23 @@ function metaTitle(pod: PodRow, workload: { kind: string; name: string } | null) .join(" · ") } +/** Inline beside the name, like the quiet status badge: absent unless the pod restarted in the window. */ +function RestartMarker({ summary }: { summary: PodRestartSummary | undefined }) { + if (!summary || summary.restarts <= 0) return null + const label = [formatRestarts(summary.restarts), summary.lastTerminatedReason].filter(Boolean).join(" · ") + return ( + + {label} + + ) +} + /** Cores read at very different magnitudes across a fleet; keep them comparable. */ const formatCores = (cores: number) => { if (!Number.isFinite(cores) || cores === 0) return "0" @@ -160,6 +179,7 @@ export function PodTable({ timeSearch, waiting, referenceTime, + restarts, }: PodTableProps) { return ( @@ -202,6 +222,7 @@ export function PodTable({ lastSeen={pod.lastSeen} referenceTime={referenceTime} /> + {podName}} - description="Pod metrics from the kubelet stats receiver." + description="Pod metrics from the kubelet stats receiver; restarts from the cluster receiver." meta={ <> {namespace && ns {namespace}} @@ -119,7 +120,7 @@ function PodDetailPage() { /> {Result.isInitial(summaryResult) ? ( - + ) : Result.isFailure(summaryResult) ? ( ) : summary ? ( - + + ) : ( @@ -171,6 +178,13 @@ function PodDetailPage() { bucketSeconds={bucketSeconds} /> + + ) diff --git a/apps/web/src/routes/infra/kubernetes/pods/index.tsx b/apps/web/src/routes/infra/kubernetes/pods/index.tsx index 02d9c2d072..f8ded8995d 100644 --- a/apps/web/src/routes/infra/kubernetes/pods/index.tsx +++ b/apps/web/src/routes/infra/kubernetes/pods/index.tsx @@ -19,7 +19,8 @@ import { InfraSetupEmpty } from "@/components/infra/infra-empty-state" import { KubernetesShell } from "@/components/infra/kubernetes/kubernetes-shell" import { PodPeekSheet } from "@/components/infra/kubernetes/pod-peek-sheet" import { PodsFilterSidebarView, type PodFilters } from "@/components/infra/k8s-filter-sidebar" -import { PodTable, PodTableLoading, podKey, type PodRow } from "@/components/infra/pod-table" +import { PodTableLoading, podKey, type PodRow } from "@/components/infra/pod-table" +import { PodTableWithRestarts } from "@/components/infra/kubernetes/pod-restarts-panel" import { FleetBand, FleetBandLoading, type FleetBandCell } from "@/components/infra/primitives/fleet-band" import { SearchToolbar, countLabel } from "@/components/common/search-toolbar" import { podFilterChips } from "@/lib/infra/pod-filter-chips" @@ -407,8 +408,10 @@ function PodsPage() { clearLabel="Clear all filters" /> ) : ( - ("PodRestartsRequest")({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + podNames: Schema.optional(StringArray), + namespace: Schema.optional(Schema.String), + limit: Schema.optional(Schema.Number), +}) {} + +export class PodRestartsResponse extends Schema.Class("PodRestartsResponse")({ + data: Schema.Array( + Schema.Struct({ + podName: Schema.String, + namespace: Schema.String, + containerName: Schema.String, + /** Restarts inside the window (last - first of the running total, floored at 0). */ + restarts: Schema.Number, + /** The running total at the window's newest sample. */ + totalRestarts: Schema.Number, + /** Why the container last terminated (OOMKilled, Error, ...); empty when it never has. */ + lastTerminatedReason: Schema.String, + }), + ), +}) {} + export class PodInfraTimeseriesRequest extends Schema.Class( "PodInfraTimeseriesRequest", )({ @@ -2848,6 +2876,13 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) + .add( + HttpApiEndpoint.post("podRestarts", "/pod-restarts", { + payload: PodRestartsRequest, + success: PodRestartsResponse, + error: queryEngineEndpointErrors, + }), + ) .add( HttpApiEndpoint.post("podInfraTimeseries", "/pod-infra-timeseries", { payload: PodInfraTimeseriesRequest, diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index ffab7a6864..821a33fa3f 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -1459,13 +1459,14 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- builder:infra:podRestartsQuery:default [5cde512b] +-- builder:infra:podRestartsQuery:default [b489a972] SELECT - metrics_gauge.ResourceAttributes['k8s.namespace.name'] AS namespace, metrics_gauge.ResourceAttributes['k8s.pod.name'] AS podName, + metrics_gauge.ResourceAttributes['k8s.namespace.name'] AS namespace, metrics_gauge.ResourceAttributes['k8s.container.name'] AS containerName, - ifNotFinite(max(metrics_gauge.Value) - min(metrics_gauge.Value), 0) AS restarts, - ifNotFinite(max(metrics_gauge.Value), 0) AS totalRestarts + ifNotFinite(greatest(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix) - argMin(metrics_gauge.Value, metrics_gauge.TimeUnix), 0), 0) AS restarts, + ifNotFinite(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix), 0) AS totalRestarts, + argMax(metrics_gauge.ResourceAttributes['k8s.container.status.last_terminated_reason'], metrics_gauge.TimeUnix) AS lastTerminatedReason FROM metrics_gauge WHERE metrics_gauge.OrgId = 'org_sql_catalog' AND metrics_gauge.TimeUnix >= '2026-01-01 10:30:00' @@ -1474,11 +1475,31 @@ SELECT AND metrics_gauge.ResourceAttributes['k8s.pod.name'] != '' AND metrics_gauge.ResourceAttributes['k8s.namespace.name'] = 'backend' AND metrics_gauge.ResourceAttributes['k8s.deployment.name'] = 'api' - GROUP BY namespace, podName, containerName - ORDER BY restarts DESC + GROUP BY podName, namespace, containerName + ORDER BY restarts DESC, totalRestarts DESC LIMIT 50 FORMAT JSON +-- builder:infra:podRestartsQuery:podNames [4976b882] +SELECT + metrics_gauge.ResourceAttributes['k8s.pod.name'] AS podName, + metrics_gauge.ResourceAttributes['k8s.namespace.name'] AS namespace, + metrics_gauge.ResourceAttributes['k8s.container.name'] AS containerName, + ifNotFinite(greatest(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix) - argMin(metrics_gauge.Value, metrics_gauge.TimeUnix), 0), 0) AS restarts, + ifNotFinite(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix), 0) AS totalRestarts, + argMax(metrics_gauge.ResourceAttributes['k8s.container.status.last_terminated_reason'], metrics_gauge.TimeUnix) AS lastTerminatedReason + FROM metrics_gauge + WHERE metrics_gauge.OrgId = 'org_sql_catalog' + AND metrics_gauge.TimeUnix >= '2026-01-01 10:30:00' + AND metrics_gauge.TimeUnix <= '2026-01-03 14:15:00' + AND metrics_gauge.MetricName = 'k8s.container.restarts' + AND metrics_gauge.ResourceAttributes['k8s.pod.name'] != '' + AND metrics_gauge.ResourceAttributes['k8s.pod.name'] IN ('api-7d9f8b6c5-x2n4k', 'worker-5c8d7f9b4-q7m2p') + GROUP BY podName, namespace, containerName + ORDER BY restarts DESC, totalRestarts DESC + LIMIT 500 + FORMAT JSON + -- builder:infra:workloadFacetsQuery:default [39bce5d0] SELECT metrics_gauge.ResourceAttributes['k8s.deployment.name'] AS name, diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index 092fb8fe70..0edf88744e 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -1232,20 +1232,6 @@ export const builderFixtures: ReadonlyArray = [ label: "default", compile: () => CH.compileUnsafe(CH.hostFilesystemsQuery({ hostName: "ip-10-0-1-42" }), window), }, - { - module: "infra", - name: "podRestartsQuery", - label: "default", - compile: () => - CH.compileUnsafe( - CH.podRestartsQuery({ - workloadKind: "deployment", - workloadName: "api", - namespace: "backend", - }), - window, - ), - }, { module: "infra", name: "hostGaugeTimeseriesQuery", @@ -1274,6 +1260,34 @@ export const builderFixtures: ReadonlyArray = [ { ...window, bucketSeconds: 300 }, ), }, + { + module: "infra", + name: "podRestartsQuery", + label: "default", + compile: () => + CH.compileUnsafe( + CH.podRestartsQuery({ + workloadKind: "deployment", + workloadName: "api", + namespace: "backend", + }), + window, + ), + }, + { + // The pod list's shape: one page of pod names, every container on them. + module: "infra", + name: "podRestartsQuery", + label: "podNames", + compile: () => + CH.compileUnsafe( + CH.podRestartsQuery({ + podNames: ["api-7d9f8b6c5-x2n4k", "worker-5c8d7f9b4-q7m2p"], + limit: 500, + }), + window, + ), + }, { module: "infra", name: "podGaugeTimeseriesQuery", diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 6efe96a111..0bff1b8248 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -591,10 +591,10 @@ export { hostGaugeTimeseriesQuery, hostNetworkTimeseriesQuery, hostFilesystemsQuery, - podRestartsQuery, listPodsQuery, listPodsSummaryQuery, podDetailSummaryQuery, + podRestartsQuery, podGaugeTimeseriesQuery, podFacetsQuery, listNodesQuery, @@ -615,8 +615,6 @@ export { type HostNetworkTimeseriesOutput, type HostFilesystemsOpts, type HostFilesystemsOutput, - type PodRestartsOpts, - type PodRestartsOutput, type ListPodsOpts, type ListPodsOutput, type ListPodsSummaryOutput, @@ -624,6 +622,8 @@ export { type SortDirection, type PodDetailSummaryOpts, type PodDetailSummaryOutput, + type PodRestartsOpts, + type PodRestartsOutput, type PodGaugeTimeseriesOpts, type PodFacetsOutput, type ListNodesOpts, diff --git a/packages/query-engine/src/ch/queries/infra.test.ts b/packages/query-engine/src/ch/queries/infra.test.ts index ea5bbfc878..d054f95c47 100644 --- a/packages/query-engine/src/ch/queries/infra.test.ts +++ b/packages/query-engine/src/ch/queries/infra.test.ts @@ -10,6 +10,7 @@ import { listPodsQuery, listPodsSummaryQuery, podDetailSummaryQuery, + podRestartsQuery, podGaugeTimeseriesQuery, podFacetsQuery, listNodesQuery, @@ -336,6 +337,43 @@ describe("podDetailSummaryQuery", () => { }) }) +describe("podRestartsQuery", () => { + it("reads the running total per container and scopes to the org and window", () => { + const { sql } = compileUnsafe(podRestartsQuery({}), baseParams) + expect(sql).toContain("OrgId = 'org_1'") + expect(sql).toContain("MetricName = 'k8s.container.restarts'") + expect(sql).toContain( + "greatest(argMax(metrics_gauge.Value, metrics_gauge.TimeUnix) - argMin(metrics_gauge.Value, metrics_gauge.TimeUnix), 0)", + ) + expect(sql).toContain("ResourceAttributes['k8s.container.status.last_terminated_reason']") + expect(sql).toMatch(/GROUP BY podName, namespace, containerName/) + expect(sql).not.toMatch(/__PARAM_\w+__/) + }) + + it("narrows to the page's pods, a namespace and a workload", () => { + const { sql } = compileUnsafe( + podRestartsQuery({ + podNames: ["api-1", "api-2"], + namespace: "backend", + workloadKind: "statefulset", + workloadName: "db", + limit: 500, + }), + baseParams, + ) + expect(sql).toContain("'api-1'") + expect(sql).toContain("'api-2'") + expect(sql).toContain("ResourceAttributes['k8s.namespace.name'] = 'backend'") + expect(sql).toContain("ResourceAttributes['k8s.statefulset.name'] = 'db'") + expect(sql).toContain("LIMIT 500") + }) + + it("skips the pod-name filter for an empty page rather than emitting IN ()", () => { + const { sql } = compileUnsafe(podRestartsQuery({ podNames: [] }), baseParams) + expect(sql).not.toMatch(/IN \(\s*\)/) + }) +}) + describe("podGaugeTimeseriesQuery", () => { it("buckets by toStartOfInterval and filters by metric name", () => { const { sql } = compileUnsafe( diff --git a/packages/query-engine/src/ch/queries/infra.ts b/packages/query-engine/src/ch/queries/infra.ts index 348bd9fd01..f395726096 100644 --- a/packages/query-engine/src/ch/queries/infra.ts +++ b/packages/query-engine/src/ch/queries/infra.ts @@ -273,54 +273,6 @@ export function hostFilesystemsQuery(opts: HostFilesystemsOpts) { .format("JSON") } -// Pod container restarts, from the k8s_cluster receiver's `k8s.container.restarts` -// (a running total per container). The window's restarts are max − min per container. - -export interface PodRestartsOpts { - podNames?: ReadonlyArray - namespace?: string - workloadKind?: "deployment" | "statefulset" | "daemonset" - workloadName?: string - limit?: number -} - -export interface PodRestartsOutput { - readonly namespace: string - readonly podName: string - readonly containerName: string - readonly restarts: number - readonly totalRestarts: number -} - -export function podRestartsQuery(opts: PodRestartsOpts) { - return from(MetricsGauge) - .select(($) => ({ - namespace: $.ResourceAttributes.get("k8s.namespace.name"), - podName: $.ResourceAttributes.get("k8s.pod.name"), - containerName: $.ResourceAttributes.get("k8s.container.name"), - restarts: CH.ifNotFinite(CH.max_($.Value).sub(CH.min_($.Value)), 0), - totalRestarts: CH.ifNotFinite(CH.max_($.Value), 0), - })) - .where(($) => [ - $.OrgId.eq(orgIdParam), - $.TimeUnix.gte(param.dateTime("startTime")), - $.TimeUnix.lte(param.dateTime("endTime")), - $.MetricName.eq("k8s.container.restarts"), - $.ResourceAttributes.get("k8s.pod.name").neq(""), - CH.when(opts.podNames, (names: ReadonlyArray) => - $.ResourceAttributes.get("k8s.pod.name").in_(...names), - ), - CH.when(opts.namespace, (v: string) => $.ResourceAttributes.get("k8s.namespace.name").eq(v)), - CH.when(opts.workloadName, (v: string) => - $.ResourceAttributes.get(workloadAttrKey(opts.workloadKind ?? "deployment")).eq(v), - ), - ]) - .groupBy("namespace", "podName", "containerName") - .orderBy(["restarts", "desc"]) - .limit(opts.limit ?? 50) - .format("JSON") -} - // Kubernetes — pod aggregations over k8s.pod.* metrics emitted by the kubelet // stats receiver. Identity carried on ResourceAttributes: // k8s.pod.name, k8s.pod.uid, k8s.namespace.name, k8s.node.name, @@ -783,6 +735,65 @@ export function podDetailSummaryQuery(opts: PodDetailSummaryOpts) { .format("JSON") } +// Pod container restarts, from the k8s_cluster receiver's `k8s.container.restarts` +// (a running total per container). The window's restarts are last - first, floored at 0: +// a kubelet that prunes dead containers can reset the total, so this undercounts, never over. + +export interface PodRestartsOpts { + podNames?: ReadonlyArray + namespace?: string + workloadKind?: "deployment" | "statefulset" | "daemonset" + workloadName?: string + limit?: number +} + +export interface PodRestartsOutput { + readonly podName: string + readonly namespace: string + readonly containerName: string + readonly restarts: number + readonly totalRestarts: number + /** `k8s.container.status.last_terminated_reason` (OOMKilled, Error, ...); empty when never terminated. */ + readonly lastTerminatedReason: string +} + +export function podRestartsQuery(opts: PodRestartsOpts) { + return from(MetricsGauge) + .select(($) => ({ + podName: $.ResourceAttributes.get("k8s.pod.name"), + namespace: $.ResourceAttributes.get("k8s.namespace.name"), + containerName: $.ResourceAttributes.get("k8s.container.name"), + restarts: CH.ifNotFinite( + CH.greatest_(CH.argMax($.Value, $.TimeUnix).sub(CH.argMin($.Value, $.TimeUnix)), CH.lit(0)), + 0, + ), + totalRestarts: CH.ifNotFinite(CH.argMax($.Value, $.TimeUnix), 0), + // The attribute rides on the resource, so the newest sample carries the newest reason. + lastTerminatedReason: CH.argMax( + $.ResourceAttributes.get("k8s.container.status.last_terminated_reason"), + $.TimeUnix, + ), + })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.TimeUnix.gte(param.dateTime("startTime")), + $.TimeUnix.lte(param.dateTime("endTime")), + $.MetricName.eq("k8s.container.restarts"), + $.ResourceAttributes.get("k8s.pod.name").neq(""), + opts.podNames?.length + ? CH.inList($.ResourceAttributes.get("k8s.pod.name"), opts.podNames) + : undefined, + CH.when(opts.namespace, (v: string) => $.ResourceAttributes.get("k8s.namespace.name").eq(v)), + CH.when(opts.workloadName, (v: string) => + $.ResourceAttributes.get(workloadAttrKey(opts.workloadKind ?? "deployment")).eq(v), + ), + ]) + .groupBy("podName", "namespace", "containerName") + .orderBy(["restarts", "desc"], ["totalRestarts", "desc"]) + .limit(opts.limit ?? 50) + .format("JSON") +} + // Pod time-series — gauge metric for one pod, optionally broken down by an // attribute key (e.g. container name, when present). diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index 786275afd6..31946e4c57 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -29,6 +29,7 @@ import type { MetricsSummaryRequest, NodeDetailSummaryRequest, PodDetailSummaryRequest, + PodRestartsRequest, PodsSummaryRequest, ServiceApdexRequest, ServiceDbEdgesForServiceRequest, @@ -447,6 +448,21 @@ export const podDetailSummary = defineQuery({ }), }) +export const podRestarts = defineQuery({ + id: "podRestarts", + profile: "aggregation", + cache: timeRangeCache, + compile: (payload: PodRestartsRequest, orgId: OrgId) => + CH.compile( + CH.podRestartsQuery({ + podNames: payload.podNames, + namespace: payload.namespace, + limit: payload.limit, + }), + { orgId, startTime: payload.startTime, endTime: payload.endTime }, + ), +}) + export const listNodes = defineQuery({ id: "listNodes", profile: "list",