Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs-site/src/content/docs/reference/cli/lifecycle.md
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,15 @@ default provider, Codex autostart setting, service state, shim state, and the re
home. Only the explicit, high-confidence Windows Orca runtime-home signature adds an actionable App-home
mismatch warning; it never changes `CODEX_HOME` automatically.

When a live proxy has already passed the identity/liveness check, `ocx status` prefers that process's
attested startup-health report for restart safety and service viability. This avoids false negatives
from a shell-local service-manager probe that lacks the running service's manager environment. The
live report is schema-validated; if it is unavailable or malformed, status falls back to the local
service and shim diagnostics. `ocx doctor` uses the same live-first rule for its **Codex restart safety**
section, so the two commands should agree on restart protection. If you are diagnosing a discrepancy,
compare the reported live startup verdict with the local service details rather than treating the shell
probe as more authoritative.

Human output also includes an **OAuth health** block after the OAuth logins summary: `OAuth health:
ok` when every known account is healthy, or `OAuth health: warning` with one redacted line per
non-healthy account (provider, masked account id, status such as reauthentication required, rate or
Expand Down
17 changes: 9 additions & 8 deletions src/cli/doctor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { homedir } from "node:os";
import { dirname, join } from "node:path";
import { getConfigDir, getConfigPath, readConfigDiagnostics } from "../config";
import { readPid } from "../config/process-state";
import { probeUncleanExitState } from "./status";
import { fetchLiveStartupHealth, probeUncleanExitState, selectStatusStartupHealth } from "./status";
import { findLiveProxy, probeHostname, type LiveProxy } from "../server/proxy-liveness";
import { directLocalHttpFetch } from "../server/direct-local-http";
import { BUN_RUNTIME_SOURCES } from "../lib/bun-runtime";
Expand Down Expand Up @@ -1354,7 +1354,14 @@ export async function runDoctor(args: string[] = []): Promise<void> {
diagnoseCodexShim(),
serviceTokenPresent,
);
const startup = collectStartupHealth(doctorConfig);
// Use the same attested live startup verdict as `ocx status` when the proxy is already
// identity-verified. A shell-local systemd probe can be a false negative for a system-wide
// service because the shell does not inherit the manager-owned environment.
const live = await findLiveProxy({
configFn: () => ({ port: doctorConfig.port, hostname: doctorConfig.hostname }),
});
const liveStartup = live ? await fetchLiveStartupHealth(live) : null;
const startup = selectStatusStartupHealth(liveStartup, () => collectStartupHealth(doctorConfig));
console.log("\nCodex restart safety");
console.log(` ${startup.rebootSafe ? "ok " : "!! "} ${startupHealthSummary(startup)}`);
console.log(` ${formatStartupRoutingDetail(startup)}`);
Expand Down Expand Up @@ -1393,12 +1400,6 @@ export async function runDoctor(args: string[] = []): Promise<void> {
}
}

// #618: identity-verified liveness first so pid-file absence does not hide a live service.
// Reuse the diagnostics config already loaded above so doctor stays read-only on malformed JSON.
const live = await findLiveProxy({
configFn: () => ({ port: doctorConfig.port, hostname: doctorConfig.hostname }),
});

// Mirrors `ocx status` through the same comparison rather than a second implementation:
// two diagnostics disagreeing about whether an install is stale is worse than one (#2701).
// No extra probe -- findLiveProxy already carried the version back.
Expand Down
93 changes: 86 additions & 7 deletions src/cli/status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ import { tokenCollidesWithAdmin } from "../lib/admin-secrets";
export { proxyHealthFailureReason, isConnectionRefused, isUncleanExitEvidence, probeUncleanExitState } from "./status-probes";
export type { ListenTarget } from "./status-probes";
import { checkProxyHealth, probeUncleanExitState, type ListenTarget } from "./status-probes";
import { LOCAL_MANAGEMENT_READ_PATHS } from "../lib/local-management-capability";
import { fetchBoundLocalManagementRead } from "../server/local-management-read-client";

/**
* The state of the data-plane admission secret the SERVICE will use. State only -- never the value.
Expand Down Expand Up @@ -233,6 +235,84 @@ function statusDashboardUrl(config: StatusListenConfig, hostname: string | undef
return `http://${dashboardHostname}:${port}/`;
}

const STARTUP_HEALTH_BOOLEAN_FIELDS = [
"routingInjected", "localRoutingDependency", "autostartEnabled", "rebootSafe",
"serviceInstalled", "serviceViable", "serviceEnabled", "serviceRunning",
"serviceStale", "serviceConflict", "shimInstalled", "shimHealthy",
"serviceSupported", "diagnosticStale",
] as const;

export async function fetchLiveStartupHealth(
live: NonNullable<Awaited<ReturnType<typeof findLiveProxy>>>,
deps: Parameters<typeof fetchBoundLocalManagementRead>[2] = {},
): Promise<StartupHealth | null> {
const result = await fetchBoundLocalManagementRead(
live, LOCAL_MANAGEMENT_READ_PATHS.startupHealth, { timeoutMs: 1_500, ...deps },
);
if (result.kind !== "response" || !result.response.ok) return null;
let payload: unknown;
try { payload = await result.response.json(); } catch { return null; }
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return null;
const row = payload as Record<string, unknown>;
if (row.status !== "native" && row.status !== "protected" && row.status !== "at-risk") return null;
if (row.protection !== "service" && row.protection !== "shim" && row.protection !== "none") return null;
if (row.routingKind !== "native" && row.routingKind !== "opencodex-local"
&& row.routingKind !== "custom-local" && row.routingKind !== "custom-remote" && row.routingKind !== "unknown") return null;
if (row.shimCoverage !== "full" && row.shimCoverage !== "cli-only" && row.shimCoverage !== "none") return null;
if (typeof row.platform !== "string") return null;
if (row.recommendedCommand !== null && typeof row.recommendedCommand !== "string") return null;
if (!row.commands || typeof row.commands !== "object" || Array.isArray(row.commands)) return null;
for (const key of ["installService", "repairService", "installShim", "restoreNative"] as const) {
if (typeof (row.commands as Record<string, unknown>)[key] !== "string") return null;
}
if (row.routingAdoption !== undefined) {
if (!row.routingAdoption || typeof row.routingAdoption !== "object" || Array.isArray(row.routingAdoption)) return null;
const adoption = row.routingAdoption as Record<string, unknown>;
if (adoption.adoption !== "not-applicable" && adoption.adoption !== "adopted"
&& adoption.adoption !== "pending-client-restart" && adoption.adoption !== "unknown") return null;
if (adoption.injectedAtMs !== null && typeof adoption.injectedAtMs !== "number") return null;
if (typeof adoption.observedClients !== "number" || !Array.isArray(adoption.staleClients)) return null;
for (const client of adoption.staleClients) {
if (!client || typeof client !== "object" || Array.isArray(client)) return null;
const row = client as Record<string, unknown>;
if (typeof row.pid !== "number" || typeof row.startedAtMs !== "number") return null;
}
}
for (const key of STARTUP_HEALTH_BOOLEAN_FIELDS) if (typeof row[key] !== "boolean") return null;
return payload as StartupHealth;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

/** Prefer an attested live verdict and evaluate the local fallback only when live state is absent. */
export function selectStatusStartupHealth(
liveStartup: StartupHealth | null,
fallback: () => StartupHealth,
): StartupHealth {
return liveStartup ?? fallback();
}

/** Build the service summary from the same startup source that `ocx status` selected. */
export function statusServiceSummary(
liveStartup: StartupHealth | null,
service: Pick<ReturnType<typeof diagnoseService>, "installed" | "summary">,
live: boolean,
): string {
if (liveStartup) {
if (liveStartup.protection === "service" && liveStartup.serviceViable) {
return `running under the live managed service (logs: ${serviceLogPath()})`;
}
const state = [
liveStartup.serviceInstalled ? "installed" : "absent",
liveStartup.serviceRunning ? "running" : "not running",
liveStartup.serviceViable ? "viable" : "not viable",
].join(", ");
const action = liveStartup.recommendedCommand ? `; run '${liveStartup.recommendedCommand}'` : "";
return `live startup reports service ${state}${action} (logs: ${serviceLogPath()})`;
}
return service.installed && !live
? `${service.summary} — registered but NOT serving; see ${serviceLogPath()} and re-run 'ocx service repair'`
: service.summary;
}

/**
* The hub block, or null when this machine is not a hub.
*
Expand Down Expand Up @@ -634,20 +714,19 @@ export async function collectStatus(): Promise<CliStatusView> {
hostname: config.hostname,
});
const bunRuntime = durableBunRuntime();
const liveStartup = live ? await fetchLiveStartupHealth(live) : null;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
const service = diagnoseService();
// A service can be registered and still not serve: the manager reports the job
// either way. `live` was already identity-probed a few lines above, so cross-check
// rather than print registration as if it were service.
const serviceSummary = service.installed && !live
? `${service.summary} — registered but NOT serving; see ${serviceLogPath()} and re-run 'ocx service repair'`
: service.summary;
// either way. When the identity-probed live proxy provides an attested startup verdict,
// prefer it over a shell-local service-manager probe that lacks the service environment.
const serviceSummary = statusServiceSummary(liveStartup, service, Boolean(live));
const codexShim = diagnoseCodexShim();
const codexShimSummary = codexShim.summary;
const startup = collectStartupHealth(config, {
const startup = selectStatusStartupHealth(liveStartup, () => collectStartupHealth(config, {
service,
shim: codexShim,
routingKind: getCodexRoutingKind(),
});
}));
const codexPlugins = diagnoseCodexBundledPlugins();
const lastClamp = loadLastEffortClamp();
const clampActive = effortClampAppliesToRuntime(lastClamp, resolvedRuntime.runtime);
Expand Down
133 changes: 133 additions & 0 deletions tests/cli/cli-status-startup-health.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
import { describe, expect, test } from "bun:test";
import { fetchLiveStartupHealth, selectStatusStartupHealth, statusServiceSummary } from "../../src/cli/status";
import type { StartupHealth } from "../../src/codex/autostart-health";

const LIVE = {
pid: 4242,
port: 10101,
hostname: "127.0.0.1",
source: "runtime" as const,
};

const SECRET = "A".repeat(43);
const NONCE = "B".repeat(43);

function startupPayload() {
return {
status: "protected",
routingKind: "opencodex-local",
routingInjected: true,
localRoutingDependency: true,
autostartEnabled: true,
rebootSafe: true,
protection: "service",
serviceInstalled: true,
serviceViable: true,
serviceEnabled: true,
serviceRunning: true,
serviceStale: false,
serviceConflict: false,
shimInstalled: true,
shimHealthy: true,
shimCoverage: "cli-only",
serviceSupported: true,
platform: "linux",
diagnosticStale: false,
recommendedCommand: null,
commands: {
installService: "ocx service install",
repairService: "ocx service repair",
installShim: "ocx codex-shim install",
restoreNative: "ocx restore",
},
};
}

function deps(body: unknown) {
return {
readRuntime: () => ({
pid: LIVE.pid,
port: LIVE.port,
hostname: LIVE.hostname,
attestationSecret: SECRET,
}),
createNonce: () => NONCE,
now: () => 1_000,
fetchImpl: async () => new Response(JSON.stringify(body), {
status: 200,
headers: { "content-type": "application/json" },
}),
};
}

describe("ocx status live startup health", () => {
test("uses an attested live startup verdict when the shell-local service probe would disagree", async () => {
const observed = await fetchLiveStartupHealth(LIVE, deps(startupPayload()));
expect(observed?.status).toBe("protected");
expect(observed?.rebootSafe).toBe(true);
expect(observed?.serviceViable).toBe(true);
expect(observed?.protection).toBe("service");
Comment thread
coderabbitai[bot] marked this conversation as resolved.
});

test("rejects malformed live startup payloads", async () => {
for (const malformed of [
{ ...startupPayload(), serviceRunning: "yes" },
(() => { const row = { ...startupPayload() } as Record<string, unknown>; delete row.platform; return row; })(),
{ ...startupPayload(), recommendedCommand: 7 },
{ ...startupPayload(), commands: { installService: "ok" } },
{ ...startupPayload(), routingAdoption: { adoption: "adopted", injectedAtMs: 1, staleClients: "bad", observedClients: 1 } },
{ ...startupPayload(), routingAdoption: { adoption: "adopted", injectedAtMs: 1, staleClients: [{ pid: "bad", startedAtMs: 1 }], observedClients: 1 } },
]) {
expect(await fetchLiveStartupHealth(LIVE, deps(malformed))).toBeNull();
}
});

test("selection prefers the attested live verdict and does not evaluate the conflicting fallback", () => {
const live = startupPayload() as StartupHealth;
let fallbackCalls = 0;
const selected = selectStatusStartupHealth(live, () => {
fallbackCalls += 1;
return { ...live, status: "at-risk", rebootSafe: false, protection: "none" } as StartupHealth;
});
expect(selected.status).toBe("protected");
expect(selected.rebootSafe).toBe(true);
expect(fallbackCalls).toBe(0);
expect(statusServiceSummary(live, { installed: false, summary: "systemd not found" }, true))
.toContain("running under the live managed service");
});

test("selection falls back to local startup diagnostics when the live read is unavailable", () => {
const local = { ...startupPayload(), status: "at-risk", rebootSafe: false, protection: "none" } as StartupHealth;
let fallbackCalls = 0;
const selected = selectStatusStartupHealth(null, () => { fallbackCalls += 1; return local; });
expect(selected).toBe(local);
expect(fallbackCalls).toBe(1);
expect(statusServiceSummary(null, { installed: true, summary: "registered" }, false))
.toContain("registered but NOT serving");
});

test("service summary never contradicts a present negative live startup verdict", () => {
const live = {
...startupPayload(),
status: "at-risk",
rebootSafe: false,
protection: "none",
serviceInstalled: false,
serviceRunning: false,
serviceViable: false,
recommendedCommand: "ocx service repair",
} as StartupHealth;
const summary = statusServiceSummary(live, { installed: true, summary: "healthy local service" }, true);
expect(summary).toContain("live startup reports service absent, not running, not viable");
expect(summary).toContain("ocx service repair");
expect(summary).not.toContain("healthy local service");
});

test("fails closed when the runtime attestation cannot bind the live PID", async () => {
const observed = await fetchLiveStartupHealth(LIVE, {
...deps(startupPayload()),
readRuntime: () => null,
});
expect(observed).toBeNull();
});
});
Loading