feat(infra): show pod restarts and last termination reason

apps/ai/src/mcp/__evals__/infra-world.ts

@@ -597,6 +597,7 @@ export const infraFixtureRules = (): FixtureRule[] => [
 				containerName: p.owner[1],
 				restarts: RESTARTS.get(p.podName) ?? 0,
 				totalRestarts: (RESTARTS.get(p.podName) ?? 0) + 2,
+				lastTerminatedReason: RESTARTS.has(p.podName) ? "OOMKilled" : "",
 			})),
 	},
 	{

apps/ai/src/mcp/tools/__tests__/infra-tools.test.ts

@@ -77,13 +77,15 @@ describe("list_infra", () => {
 					containerName: "checkout",
 					restarts: 1,
 					totalRestarts: 3,
+					lastTerminatedReason: "",
 				},
 				{
 					namespace: "staging",
 					podName: INFRA_WORLD.oomPod,
 					containerName: "checkout",
 					restarts: 7,
 					totalRestarts: 9,
+					lastTerminatedReason: "",
 				},
 			],
 		})

apps/api/src/routes/internal/query-engine.http.ts

@@ -54,6 +54,7 @@ import {
 	ListPodsResponse,
 	PodsSummaryResponse,
 	PodDetailSummaryResponse,
+	PodRestartsResponse,
 	PodInfraTimeseriesResponse,
 	PodFacetsResponse,
 	ListContainersResponse,
@@ -1562,6 +1563,22 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query
 						})
 					}),
 				)
+				.handle("podRestarts", ({ payload }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						const rows = yield* runQuery(Queries.podRestarts, tenant, payload)
+						return new PodRestartsResponse({
+							data: rows.map((row) => ({
+								podName: row.podName,
+								namespace: row.namespace,
+								containerName: row.containerName,
+								restarts: Number(row.restarts) || 0,
+								totalRestarts: Number(row.totalRestarts) || 0,
+								lastTerminatedReason: row.lastTerminatedReason,
+							})),
+						})
+					}),
+				)
 				.handle("podInfraTimeseries", ({ payload }) =>
 					Effect.gen(function* () {
 						const tenant = yield* CurrentTenant.Context

apps/web/src/api/warehouse/infra.ts

@@ -6,6 +6,7 @@ import {
 	ListPodsRequest,
 	PodsSummaryRequest,
 	PodDetailSummaryRequest,
+	PodRestartsRequest,
 	PodInfraTimeseriesRequest,
 	PodFacetsRequest,
 	ListNodesRequest,
@@ -28,6 +29,7 @@ import {
 	type ListPodsResponse,
 	type PodsSummaryResponse,
 	type PodDetailSummaryResponse,
+	type PodRestartsResponse,
 	type PodInfraTimeseriesResponse,
 	type PodFacetsResponse,
 	type ListNodesResponse,
@@ -354,6 +356,32 @@ export function podDetailSummary({ data }: { data: PodDetailSummaryInput }) {
 	)
 }
 
+export interface PodRestartsInput {
+	startTime: string
+	endTime: string
+	podNames?: ReadonlyArray<string>
+	namespace?: string
+	limit?: number
+}
+
+export function podRestarts({ data }: { data: PodRestartsInput }) {
+	return runWarehouseQuery("podRestarts", () =>
+		Effect.gen(function* () {
+			const client = yield* MapleInternalAtomClient
+			const response: PodRestartsResponse = yield* client.queryEngine.podRestarts({
+				payload: new PodRestartsRequest({
+					startTime: data.startTime,
+					endTime: data.endTime,
+					podNames: data.podNames,
+					namespace: data.namespace,
+					limit: data.limit,
+				}),
+			})
+			return response
+		}),
+	)
+}
+
 export type PodInfraMetric = "cpu_usage" | "cpu_limit" | "cpu_request" | "memory_limit" | "memory_request"
 
 export interface PodInfraTimeseriesInput {

apps/web/src/components/infra/kubernetes/pod-restarts-panel.tsx

@@ -0,0 +1,128 @@
+import type { ComponentProps } from "react"
+import { EMPTY_VALUE } from "@maple/ui/lib/format"
+import { TONE_TEXT } from "@maple/ui/lib/tone"
+import { cn } from "@maple/ui/lib/utils"
+
+import { ColumnHead, DataTable } from "@/components/common/data-table"
+import { SectionHeading } from "@/components/common/section-heading"
+import { StatRailItem, StatRailItemSkeleton } from "@/components/common/stat-rail"
+import { PodTable } from "@/components/infra/pod-table"
+import { restartTone, summarizePodRestarts, type PodRestartRow } from "@/components/infra/pod-restarts"
+import { Result, useAtomValue } from "@/lib/effect-atom"
+import { podRestartsResultAtom } from "@/lib/services/atoms/warehouse-query-atoms"
+
+/** A page of 50 pods with a few containers each, with headroom for sidecars. */
+const PAGE_RESTARTS_LIMIT = 500
+
+interface PodWindow {
+	podName: string
+	namespace?: string
+	startTime: string
+	endTime: string
+}
+
+const podRestartsAtom = ({ podName, namespace, startTime, endTime }: PodWindow) =>
+	podRestartsResultAtom({ data: { podNames: [podName], namespace, startTime, endTime } })
+
+/**
+ * The pod table with each row's restarts beside its name. Restarts come from
+ * the cluster collector, not the kubelet, so they load as a second query and
+ * the table renders without them until it lands. Render only with a non-empty page.
+ */
+export function PodTableWithRestarts({
+	startTime,
+	endTime,
+	...props
+}: Omit<ComponentProps<typeof PodTable>, "restarts"> & { startTime: string; endTime: string }) {
+	const result = useAtomValue(
+		podRestartsResultAtom({
+			data: {
+				startTime,
+				endTime,
+				podNames: props.pods.map((pod) => pod.podName),
+				limit: PAGE_RESTARTS_LIMIT,
+			},
+		}),
+	)
+	const restarts = Result.builder(result)
+		.onSuccess((response) => summarizePodRestarts(response.data))
+		.orElse(() => undefined)
+	return <PodTable {...props} restarts={restarts} />
+}
+
+/** The restarts tile for the pod page's stat rail. */
+export function PodRestartsStat(props: PodWindow) {
+	const result = useAtomValue(podRestartsAtom(props))
+	return Result.builder(result)
+		.onInitial(() => <StatRailItemSkeleton />)
+		.onError(() => <StatRailItem eyebrow="Restarts" value={EMPTY_VALUE} hint="unavailable" compact />)
+		.onSuccess((response) => {
+			// No rows means the cluster collector isn't reporting this pod, not zero restarts.
+			if (response.data.length === 0) {
+				return <StatRailItem eyebrow="Restarts" value={EMPTY_VALUE} hint="not reported" compact />
+			}
+			const summary = summarizePodRestarts(response.data).values().next().value
+			const restarts = summary?.restarts ?? 0
+			const reason = summary?.lastTerminatedReason ?? ""
+			return (
+				<StatRailItem
+					eyebrow="Restarts"
+					value={String(restarts)}
+					hint={`of ${summary?.totalRestarts ?? 0} total`}
+					subline={reason ? `Last terminated: ${reason}` : undefined}
+					tone={restartTone(restarts, reason)}
+					compact
+				/>
+			)
+		})
+		.render()
+}
+
+/** Per-container breakdown, shown once any container has restarted or terminated. */
+export function PodContainerRestarts(props: PodWindow) {
+	const result = useAtomValue(podRestartsAtom(props))
+	const rows = Result.builder(result)
+		.onSuccess((response) => response.data)
+		.orElse(() => [])
+	if (!rows.some((row) => row.totalRestarts > 0 || row.lastTerminatedReason !== "")) return null
+	return (
+		<div className="space-y-3">
+			<SectionHeading as="h3" title="Container restarts" hint="most restarts first" />
+			<DataTable.Root ariaLabel="Container restarts">
+				<DataTable.Head>
+					<ColumnHead label="Container" width="w-0 flex-1 min-w-[200px]" />
+					<ColumnHead label="In window" align="right" width="w-[96px]" />
+					<ColumnHead label="Total" align="right" width="w-[96px]" />
+					<ColumnHead label="Last terminated" align="right" width="w-[160px]" />
+				</DataTable.Head>
+				{rows.map((row) => (
+					<ContainerRow key={row.containerName} row={row} />
+				))}
+			</DataTable.Root>
+		</div>
+	)
+}
+
+function ContainerRow({ row }: { row: PodRestartRow }) {
+	return (
+		<div className="flex items-center gap-4 border-b border-border/40 px-4 py-3 last:border-0">
+			<span className="w-0 min-w-[200px] flex-1 truncate font-mono text-xs text-foreground">
+				{row.containerName || EMPTY_VALUE}
+			</span>
+			<span
+				className={cn(
+					"w-[96px] text-right font-mono text-xs tabular-nums",
+					TONE_TEXT[restartTone(row.restarts, row.lastTerminatedReason)],
+				)}
+			>
+				{row.restarts}
+			</span>
+			<span className="w-[96px] text-right font-mono text-xs tabular-nums text-muted-foreground">
+				{row.totalRestarts}
+			</span>
+			<span className="w-[160px] truncate text-right text-xs text-muted-foreground">
+				{row.lastTerminatedReason || EMPTY_VALUE}
+			</span>
+		</div>
+	)
+}

apps/web/src/components/infra/pod-restarts.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, it } from "vitest"
+import { formatRestarts, restartTone, summarizePodRestarts, type PodRestartRow } from "./pod-restarts"
+
+const row = (overrides: Partial<PodRestartRow>): PodRestartRow => ({
+	podName: "api-1",
+	namespace: "backend",
+	containerName: "app",
+	restarts: 0,
+	totalRestarts: 0,
+	lastTerminatedReason: "",
+	...overrides,
+})
+
+describe("summarizePodRestarts", () => {
+	it("sums restarts across a pod's containers", () => {
+		const summary = summarizePodRestarts([
+			row({ containerName: "app", restarts: 2, totalRestarts: 5 }),
+			row({ containerName: "sidecar", restarts: 1, totalRestarts: 1 }),
+		])
+		expect(summary.get("backend/api-1")).toEqual({
+			restarts: 3,
+			totalRestarts: 6,
+			lastTerminatedReason: "",
+		})
+	})
+
+	it("takes the reason from the container that restarted most in the window", () => {
+		const summary = summarizePodRestarts([
+			row({ containerName: "sidecar", restarts: 1, totalRestarts: 9, lastTerminatedReason: "Error" }),
+			row({ containerName: "app", restarts: 4, totalRestarts: 4, lastTerminatedReason: "OOMKilled" }),
+		])
+		expect(summary.get("backend/api-1")?.lastTerminatedReason).toBe("OOMKilled")
+	})
+
+	it("falls back to any container with a reason when the top one has none", () => {
+		const summary = summarizePodRestarts([
+			row({ containerName: "app", restarts: 3, lastTerminatedReason: "" }),
+			row({ containerName: "init", restarts: 0, totalRestarts: 1, lastTerminatedReason: "Completed" }),
+		])
+		expect(summary.get("backend/api-1")?.lastTerminatedReason).toBe("Completed")
+	})
+
+	it("keeps same-named pods in different namespaces apart", () => {
+		const summary = summarizePodRestarts([
+			row({ namespace: "backend", restarts: 1 }),
+			row({ namespace: "staging", restarts: 7 }),
+		])
+		expect(summary.get("backend/api-1")?.restarts).toBe(1)
+		expect(summary.get("staging/api-1")?.restarts).toBe(7)
+	})
+
+	it("returns an empty map for no rows", () => {
+		expect(summarizePodRestarts([]).size).toBe(0)
+	})
+})
+
+describe("restartTone", () => {
+	it("is neutral without restarts in the window, even with an old OOM kill", () => {
+		expect(restartTone(0, "OOMKilled")).toBe("neutral")
+	})
+
+	it("reads an OOM kill as critical and other restarts as a warning", () => {
+		expect(restartTone(2, "OOMKilled")).toBe("crit")
+		expect(restartTone(2, "Error")).toBe("warn")
+		expect(restartTone(1, "")).toBe("warn")
+	})
+})
+
+describe("formatRestarts", () => {
+	it("pluralizes", () => {
+		expect(formatRestarts(1)).toBe("1 restart")
+		expect(formatRestarts(3)).toBe("3 restarts")
+	})
+})

apps/web/src/components/infra/pod-restarts.ts

@@ -0,0 +1,54 @@
+import type { PodRestartsResponse } from "@maple/domain/http"
+import type { Tone } from "@maple/ui/lib/tone"
+
+export type PodRestartRow = PodRestartsResponse["data"][number]
+
+export interface PodRestartSummary {
+	/** Restarts inside the window, summed over the pod's containers. */
+	readonly restarts: number
+	/** Running totals summed over the pod's containers. */
+	readonly totalRestarts: number
+	/** The reason from the container that restarted most, or any container that has one. */
+	readonly lastTerminatedReason: string
+}
+
+/** Row identity. A pod name repeats across namespaces; the pair doesn't. */
+export const podKey = (pod: { namespace: string; podName: string }) => `${pod.namespace}/${pod.podName}`
+
+const byMostRestarts = (a: PodRestartRow, b: PodRestartRow) =>
+	b.restarts - a.restarts || b.totalRestarts - a.totalRestarts
+
+/** Folds per-container rows into one summary per pod, keyed by `podKey`. */
+export function summarizePodRestarts(
+	rows: ReadonlyArray<PodRestartRow>,
+): ReadonlyMap<string, PodRestartSummary> {
+	const byPod = new Map<string, Array<PodRestartRow>>()
+	for (const row of rows) {
+		const key = podKey(row)
+		const containers = byPod.get(key)
+		if (containers) containers.push(row)
+		else byPod.set(key, [row])
+	}
+	return new Map(
+		[...byPod].map(([key, containers]) => {
+			const ranked = [...containers].sort(byMostRestarts)
+			return [
+				key,
+				{
+					restarts: ranked.reduce((sum, c) => sum + c.restarts, 0),
+					totalRestarts: ranked.reduce((sum, c) => sum + c.totalRestarts, 0),
+					lastTerminatedReason:
+						ranked.find((c) => c.lastTerminatedReason !== "")?.lastTerminatedReason ?? "",
+				},
+			]
+		}),
+	)
+}
+
+/** An OOM kill is the one reason that says "raise the limit", so it reads as critical. */
+export function restartTone(restarts: number, reason: string): Tone {
+	if (restarts <= 0) return "neutral"
+	return reason === "OOMKilled" ? "crit" : "warn"
+}
+
+export const formatRestarts = (n: number) => `${n} ${n === 1 ? "restart" : "restarts"}`