From 91e4e6e8f686805fb64cc23b36df75f464eb34c5 Mon Sep 17 00:00:00 2001 From: zy95-12 <135135297+zy95-12@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:52:32 +0800 Subject: [PATCH] Reconcile lost SDK controllers with durable stop and process exit evidence --- PROJECT.md | 53 +++-------- ROADMAP.md | 2 + changelog.d/2483-controller-exit.fixed.md | 1 + core/src/host/project-harness-worker.mjs | 12 ++- core/src/host/project-harness.js | 71 ++++++++++++-- core/src/host/project-resources.js | 1 + core/src/host/project-sdk-exit.mjs | 79 +++++++++++++++ core/test/project-harness.test.js | 76 ++++++++++++++- core/test/project-sdk-exit-sdk.test.js | 44 +++++++++ core/test/project-sdk-exit.test.js | 95 +++++++++++++++++++ ...026-11-05-controller-loss-exit-evidence.md | 29 ++++++ docs/backend-operations.md | 9 ++ ...1-05-controller-exit-memory-checkpoints.md | 45 +++++++++ docs/openapi/flybot.openapi.json | 10 +- docs/subsystems/backend-service.md | 4 + docs/subsystems/project-harness.md | 23 +++++ docs/subsystems/project-resources.md | 6 ++ .../files/src/host/project-harness-worker.mjs | 12 ++- .../ds-bot/files/src/host/project-harness.js | 71 ++++++++++++-- .../files/src/host/project-resources.js | 1 + .../files/src/host/project-sdk-exit.mjs | 79 +++++++++++++++ .../ds-bot/files/test/project-harness.test.js | 76 ++++++++++++++- .../files/test/project-sdk-exit-sdk.test.js | 44 +++++++++ .../files/test/project-sdk-exit.test.js | 95 +++++++++++++++++++ 24 files changed, 873 insertions(+), 65 deletions(-) create mode 100644 changelog.d/2483-controller-exit.fixed.md create mode 100644 core/src/host/project-sdk-exit.mjs create mode 100644 core/test/project-sdk-exit-sdk.test.js create mode 100644 core/test/project-sdk-exit.test.js create mode 100644 decisions/2026-11-05-controller-loss-exit-evidence.md create mode 100644 docs/history/2026-11-05-controller-exit-memory-checkpoints.md create mode 100644 patches/ds-bot/files/src/host/project-sdk-exit.mjs create mode 100644 patches/ds-bot/files/test/project-sdk-exit-sdk.test.js create mode 100644 patches/ds-bot/files/test/project-sdk-exit.test.js diff --git a/PROJECT.md b/PROJECT.md index 3d0870d32..c58579c64 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -1,5 +1,7 @@ # FlyBot — Project State +#2483 controller-loss recovery joins a resource-side settled stop receipt with exact process exit, or existing durable host exit evidence. Reconcile worker and lease together; physical exit grants no Task completion or next-role decision. + #2483 restart recovery notifies the resident Chief through existing durable inspection requests. Current exit/retirement facts wake judgement; Chief chooses recovery and unknown exits remain held. Endpoint tests alone do not prove this notification chain. #2483: Developer Git mutation rights follow its bound isolated resource clone, independently of App credentials. Shared host Git metadata and read-only role policies keep their existing boundaries. @@ -28,46 +30,17 @@ architecture lives in [docs/architecture.md](docs/architecture.md) and Current stage and priorities: [ROADMAP.md](ROADMAP.md). How decisions are made: [docs/governance.md](docs/governance.md). Dated decisions: [decisions/](decisions/README.md). -## SDK authentication — 2026-11-04 - -#2398 binds readiness and setup to the existing resource profile, preserving its -paths and Task-role context. Known SDK authentication failures reach authRequired -without becoming code verdicts. No OAuth broker, credential copying or live login. - -## Role App delivery — 2026-11-04 - -#2365 opts Project roles into explicit App connections and dedicated short-lived -credential helpers. Original candidate Effects observe SDK commits/PRs without -resealing; actual App Review and host merge/Chief acceptance stay separate. -No Owner fallback or live migration. [Decision](decisions/2026-11-04-role-github-apps.md) -and [operator contract](docs/role-github-apps.md); live App acceptance requires authorized installations. - -## Follow-up reliability delivery — 2026-11-04 - -Owner authorizes sequential #2336/#2337/#2334/#2346/#2338/#2347/#2331/#2342 -implementation, one PR each, tests/memory/docs and validated merge to the delivery -branch. #2336 establishes owned reservation rollback and a durable pre-spawn fence; -unknown execution, workspace/context and current authority remain protected. -#2337 validates the actual SDK-resolved CLI and role profile before inference; -configuration readiness is separate from model/test/exit evidence. -#2334 separates executor dependency readiness from host test outcomes and preserves -same-candidate retry evidence; remote unknown exit retains preparation occupancy. -#2346 admits cwd-only Task-role restoration with current binding/exit proof and -explicit resumed/reconstructed identity; clearing bindings never resets context. -No live task/configuration/credential mutation or deployment is implied. - -## Execution recovery — 2026-11-04 - -#2325 aligns actual reused SDK/resource checkouts with pinned baseline/seed facts, -preserves drafts and admits explicit inspect/integrate inputs through existing -Effects. Chief selects only approved repository sources. #2327 adds bounded, durable Chief protocol correction in the resident Session; -exhaustion retains incomplete judgement without automatic escalation. Follow-ups: -#2328 adds confirmed-idle wait suspension and FIFO continuation re-admission -through the original queue; exit-unknown capacity remains held. #2330 adds durable scheduled inspection reconciliation, configured deadlines, -exit-proven bounded recovery and health visibility; independent Task refusals do not -abort other decisions. No automatic business escalation or live rollout. See -[delivery contract](docs/subsystems/project-delivery.md). Live Task mutation and -deployment remain separate from source/test delivery. +## Runtime and recovery contracts + +Earlier implementation checkpoints are [preserved verbatim](docs/history/2026-11-05-controller-exit-memory-checkpoints.md). +Current resource profile identity, Task-role context, CLI/configuration validation and +owned pre-spawn reservations remain binding. Unknown worker/preparation exit keeps +occupancy; dependency readiness and exact-candidate test evidence remain separate. +Apps require explicit installations and scoped credentials with no Owner fallback; +SDK candidate/PR adoption, App Review, host merge and Chief acceptance remain distinct. +Recovery/correction reuses original queues and schedules, preserves drafts and current +rights, and never chooses automatic business escalation. All canonical subsystem +contracts remain active. No live task/config/credential change or deployment is implied. ## Established backend foundations diff --git a/ROADMAP.md b/ROADMAP.md index 03f9f96ec..517017681 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,7 @@ # FlyBot — Roadmap +#2483 controller loss: verify local/Docker worker shutdown and same-generation lease reconciliation; legacy records need actual saved exit evidence. Explicit Chief baseline choice remains the next independent stage. + #2483 restart: cover saved state reload through Chief choice and actual pool re-dispatch, including historical blocked claims, preserved drafts and exit uncertainty. No fixed recovery role, automatic Task acceptance or live deployment. #2483 resource Git parity: verify current Project/Attempt/resource/cwd and private Git metadata before granting Dev its clone `.git`; validate actual SDK mutations and read-only denial separately from live resource acceptance. diff --git a/changelog.d/2483-controller-exit.fixed.md b/changelog.d/2483-controller-exit.fixed.md new file mode 100644 index 000000000..835e8e86b --- /dev/null +++ b/changelog.d/2483-controller-exit.fixed.md @@ -0,0 +1 @@ +Reconcile lost SDK controllers using verified resource-side stop/process evidence and repair partial worker/lease releases while preserving unknown-exit fences. diff --git a/core/src/host/project-harness-worker.mjs b/core/src/host/project-harness-worker.mjs index de837d66b..2f6265e6f 100644 --- a/core/src/host/project-harness-worker.mjs +++ b/core/src/host/project-harness-worker.mjs @@ -6,9 +6,14 @@ import { readFileSync, existsSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { runHarnessTurn } from './project-harness-turn.js' +import { registerSdkExit, acknowledgeSdkStop, SDK_EXIT_VERSION } from './project-sdk-exit.mjs' +const exitIdentity = process.argv[2] ? JSON.parse(process.argv[2]) : null +if (exitIdentity) await registerSdkExit(exitIdentity) const input = createInterface({ input: process.stdin, crlfDelay: Infinity }) let githubRuntime -const write = value => process.stdout.write(JSON.stringify(githubRuntime ? githubRuntime.redact(value) : value) + '\n') +let outputClosed = false +const write = value => { if (!outputClosed) process.stdout.write(JSON.stringify(githubRuntime ? githubRuntime.redact(value) : value) + '\n') } +process.stdout.on('error', () => { outputClosed = true; void shutdown() }) const answers = new Map(), credentials = new Map() let controller, sessionId, settings, busy = false, shuttingDown = false const completedTurns = new Map() @@ -16,6 +21,7 @@ const permission = request => new Promise(resolve => { const id = randomUUID(); answers.set(id, resolve); write({ type: 'permission', id, request }) }) async function turn(message) { + if (shuttingDown) return if (message.turnId && completedTurns.has(message.turnId)) { write(completedTurns.get(message.turnId)); return } if (busy) { write({ type: 'failure', reason: 'SDK Session is already answering' }); return } busy = true; controller = new AbortController() @@ -58,10 +64,12 @@ async function shutdown() { for (const resolve of answers.values()) resolve({ allow: false }) for (let n = 0; busy && n < 100; n++) await new Promise(resolve => setTimeout(resolve, 50)) if (busy) return // No exit claim while the SDK has not settled; host retains capacity. + if (exitIdentity) await acknowledgeSdkStop(exitIdentity) write({ type: 'stopped', released: true }) input.close(); process.stdin.pause(); setTimeout(() => process.exit(0), 50) } input.on('close', () => { void shutdown() }) +process.on('SIGTERM', () => { void shutdown() }) function sdkVersion(name) { try { let directory = dirname(fileURLToPath(import.meta.resolve(name))) @@ -73,4 +81,4 @@ function sdkVersion(name) { } catch { /* Missing SDK is an explicit failed handshake, not a simulated version. */ } return null } -write({ type: 'ready', roleGithub: 'role-github/v1', receiptCorrection: 'input-correction/v1', pid: process.pid, sdk: { codex: sdkVersion('@openai/codex-sdk'), 'claude-agent': sdkVersion('@anthropic-ai/claude-agent-sdk') } }) +write({ type: 'ready', roleGithub: 'role-github/v1', receiptCorrection: 'input-correction/v1', exitRecovery: exitIdentity ? SDK_EXIT_VERSION : null, pid: process.pid, sdk: { codex: sdkVersion('@openai/codex-sdk'), 'claude-agent': sdkVersion('@anthropic-ai/claude-agent-sdk') } }) diff --git a/core/src/host/project-harness.js b/core/src/host/project-harness.js index edaf68d6e..842eaf125 100644 --- a/core/src/host/project-harness.js +++ b/core/src/host/project-harness.js @@ -5,7 +5,7 @@ import { roleGithubRuntime } from './project-role-github-runtime.mjs' // Harness worker lifecycle plugs into the existing Workflow. It never accepts a Task. import { randomUUID } from 'node:crypto' import { createHash } from 'node:crypto' -import { mkdir, appendFile, readFile, copyFile, writeFile } from 'node:fs/promises' +import { mkdir, appendFile, readFile, copyFile, writeFile, realpath, lstat } from 'node:fs/promises' import { join, resolve, relative, dirname } from 'node:path' import { createInterface } from 'node:readline' import { verifyRolePolicy, roleExecutionPrompt } from './project-role-policy.js' @@ -13,10 +13,11 @@ import { sdkContinuationArchive } from './project-session-continuation.js' import { fileDigest } from './project-resource-workspaces.js' import { copyDependencyCacheScript, executionOutcome } from './project-execution-environment.js' import { gitWritableRoots } from './project-git-sandbox.js' +import { SDK_EXIT_VERSION } from './project-sdk-exit.mjs' import { ownEntry, setEntry } from './store.js' import { checkContextPrompt, resolveContextBudget } from './project-context-budget.js' import { requireExecutionAuthority } from './project-execution-authority.js' -import { taskSession, backendBindingKey, backendContextKey, resumableBinding, roleDefinitionKey } from './project-task-sessions.js' +import { taskSession, backendBindingKey, backendContextKey, resumableBinding, roleDefinitionKey, confirmedWorkerExit } from './project-task-sessions.js' const refuse = message => Object.assign(new Error(message), { code: 'bot/harness' }) const sha = text => createHash('sha256').update(text).digest('hex') const VERSIONS = { codex: '0.166.0', 'claude-agent': '0.3.316' } @@ -151,7 +152,7 @@ export function createProjectHarness(rt, { roleGithub } = {}) { const row = { id, ...(conversation?.roomId ? { roomId: conversation.roomId } : {}), model: session.execution.model ?? selected.selected.model ?? null, chiefRunId: chief?.runId ?? null, chiefInvocation: chief?.runId ? rt.state.projectExecutions?.[chief.runId]?.invocation ?? null : null, visibility: conversation?.visibility ?? 'project', ...(conversation?.visibility === 'private' ? { memberId: conversation.memberId } : {}), projectId: session.projectId, taskId: session.taskId, attemptId: session.attemptId, role: session.role, policyRole: session.policyRole ?? session.definition?.policyRole ?? session.role, sessionKey: session.id, resourceId: session.resourceLease.resourceId, leaseId: session.resourceLease.id, generation: session.resourceLease.generation, harness: session.execution.harness, sdkVersion: null, expectedSdkVersion: VERSIONS[session.execution.harness], principalId: selected.principalId, profile: selected.profile, - cwd: workspace.path, scratchDirectory, dependencyPreparationId: session.environmentPreparation?.id ?? null, spawnPhase: 'registered', status: 'prepared', executionStartedAt: null, createdAt: Date.now(), eventCount: 0 } + cwd: workspace.path, scratchDirectory, exitRecovery: { version: SDK_EXIT_VERSION }, dependencyPreparationId: session.environmentPreparation?.id ?? null, spawnPhase: 'registered', status: 'prepared', executionStartedAt: null, createdAt: Date.now(), eventCount: 0 } if (logical) Object.assign(row, { logicalSessionId: logical.id, bindingKey, bindingVersion: 'task-role-binding/v3', contextIdentityKey: backendContextKey({ ...identity, definition: definitionKey }), definitionKey, ...(previous ? { ...(previous.restoration?.mode === 'reconstructed' ? {} : { backendSessionId: previous.sessionId }), ...(previous.restoration ? { restoration: previous.restoration } : {}), continuedFrom: previous.workerId, ...(previous.key !== bindingKey ? { migratedBindingKey: previous.key } : {}) } : { rotationReason: 'First Task-role execution' }) }) else if (chief) Object.assign(row, { logicalSessionId: chief.agentId, bindingKey, ...(chief.resumeSessionId ? { backendSessionId: chief.resumeSessionId, continuedFrom: chief.previousWorkerId } : {}) }) setEntry(workers(), id, row) @@ -185,7 +186,8 @@ export function createProjectHarness(rt, { roleGithub } = {}) { if (row.restoration) session.restoration = structuredClone(row.restoration) } await rt.projectResources.fenceSpawn(row.leaseId, row.generation, id) - const process = rt.projectResources.startProcess(row.projectId, row.resourceId, { argv: ['node', workerPath], cwd: workspace.path }) + const exitIdentity = { version: SDK_EXIT_VERSION, root: selected.resource.root, workerId: id, leaseId: row.leaseId, generation: row.generation } + const process = rt.projectResources.startProcess(row.projectId, row.resourceId, { argv: ['node', workerPath, JSON.stringify(exitIdentity)], cwd: workspace.path }) const control = { process, chief, githubScope, contextBudget, resumeSessionId: chief?.resumeSessionId ?? (previous?.restoration?.mode === 'reconstructed' ? undefined : previous?.sessionId), continuedPolicyHash: previous?.policySha256, archivePath, continuationPath, logical, bindingKey, env: selected.env, profile: { ...selected.selected, ...(session.execution.model ? { model: session.execution.model } : {}) }, session, prompt: spec.prompt, outputs: [], closed: false, ready: false } row.spawnPhase = 'spawned' active.set(id, control) @@ -204,6 +206,7 @@ export function createProjectHarness(rt, { roleGithub } = {}) { try { const signal = JSON.parse(line), lease = ownEntry(rt.state.projectResourceLeases ?? {}, row.leaseId) if (signal.type === 'ready' && active.get(id) === control) { + row.exitRecovery.supported = signal.exitRecovery === SDK_EXIT_VERSION control.receiptCorrectionSupported = signal.receiptCorrection === 'input-correction/v1' control.readyObserved = true control.observedSdkVersion = /^\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?$/.test(signal.sdk?.[row.harness] ?? '') ? signal.sdk[row.harness] : null @@ -496,12 +499,65 @@ export function createProjectHarness(rt, { roleGithub } = {}) { await rt.save() return { ok: true } } + async function exitReceipt(row, action) { + const lease = ownEntry(rt.state.projectResourceLeases ?? {}, row.leaseId) + const resource = rt.projectResources.requireBinding(row.projectId, lease) + if (lease.projectId !== row.projectId || lease.attemptId !== row.attemptId || lease.resourceId !== row.resourceId || lease.role !== row.role || lease.generation !== row.generation || lease.worker?.handle !== row.id) throw refuse('SDK exit lease binding changed') + const identity = { version: SDK_EXIT_VERSION, root: resource.root, workerId: row.id, leaseId: row.leaseId, generation: row.generation } + const helper = join(resource.runtimeDir ?? resolve(import.meta.dirname, '../..'), 'src/host/project-sdk-exit.mjs') + const result = await rt.projectResources.command(row.projectId, row.resourceId, { cwd: resource.root, argv: ['node', helper, action, JSON.stringify(identity)], timeout: 10000 }) + return { lease, identity, result: JSON.parse(result.stdout) } + } + async function durableHostStop(row) { + if (row.exit?.code !== 0 || row.exit.signal || !row.eventLog || !row.lastEventHash) return false + try { + const root = await realpath(join(rt.home, 'project-artifacts', row.attemptId)), path = await realpath(row.eventLog), info = await lstat(row.eventLog) + if (!path.startsWith(root + '/') || path !== row.eventLog || !info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024) return false + const event = JSON.parse((await readFile(path, 'utf8')).trimEnd().split('\n').at(-1)) + return event.type === 'stopped' && event.released === true && sha(JSON.stringify(event)) === row.lastEventHash + } catch { return false } + } + async function reconcileExit(id, { stop = false } = {}) { + const row = ownEntry(workers(), id) + if (!row || active.has(id) && !active.get(id).closed) return false + const savedLease = ownEntry(rt.state.projectResourceLeases ?? {}, row.leaseId) + if (row.status === 'released' && savedLease?.status === 'released' && (confirmedWorkerExit(rt.state, row) || row.releaseProof?.kind === 'sdk-worker-spawn-failed' && row.releaseProof.workerId === id && row.releaseProof.generation === row.generation && savedLease.releaseProof?.kind === row.releaseProof.kind)) return true + let observed, source = 'resource-stopped-receipt' + try { + const persisted = row.status === 'released' && row.releaseProof?.kind === 'sdk-stop-ack-and-process-exit' && row.releaseProof.workerId === id && row.releaseProof.generation === row.generation + if (persisted || await durableHostStop(row)) { + const lease = ownEntry(rt.state.projectResourceLeases ?? {}, row.leaseId) + rt.projectResources.requireBinding(row.projectId, lease) + if (lease.projectId !== row.projectId || lease.attemptId !== row.attemptId || lease.resourceId !== row.resourceId || lease.role !== row.role || lease.worker?.handle !== id) return false + const identity = { version: SDK_EXIT_VERSION, workerId: id, leaseId: row.leaseId, generation: row.generation } + observed = { lease, identity, result: { ...identity, status: 'exited' } }; source = persisted ? 'persisted-sdk-exit-proof' : 'durable-host-stop-event' + } else if (row.exitRecovery?.version === SDK_EXIT_VERSION) observed = await exitReceipt(row, stop ? 'stop' : 'inspect') + else return false + } catch { return false } + const { lease, identity, result } = observed + if (row !== ownEntry(workers(), id) || lease !== ownEntry(rt.state.projectResourceLeases ?? {}, identity.leaseId) || row.leaseId !== identity.leaseId || row.generation !== identity.generation || lease.generation !== identity.generation || lease.worker?.handle !== id || active.has(id) && !active.get(id).closed || Object.entries(identity).some(([key, value]) => key !== 'root' && result[key] !== value) || result.status !== 'exited') return false + rt.projectResources.requireBinding(row.projectId, lease) + const proof = { kind: 'sdk-stop-ack-and-process-exit', source, workerId: id, generation: row.generation, at: Date.now() } + row.status = 'released'; row.releasedAt = proof.at; row.releaseProof = proof + if (row.githubAccess) { row.githubAccess.renewing = false; row.githubAccess.status = 'closed' } + if (row.scratchDirectory) row.scratchCleanupPending = true + clearTimeout(active.get(id)?.timer); active.delete(id) + lease.releaseProof = proof + await rt.projectResources.release(lease.id, lease.generation, { handle: id, exited: true, proof }) + await rt.projectDelivery.releaseSession(id) + await rt.save() + try { await exitReceipt(row, 'forget') } catch { /* Durable exit proof is retained even if artifact cleanup fails. */ } + return true + } + async function reconcileExits() { + for (const row of Object.values(workers())) await reconcileExit(row.id) + } async function release(id) { const row = ownEntry(workers(), id) if (!row) return false - if (row.status === 'released') return true + if (row.status === 'released') { await reconcileExit(id); return true } const control = active.get(id) - if (!control) throw refuse('SDK worker exit cannot be confirmed after restart') + if (!control || control.closed && !control.stoppedConfirmed && !control.spawnFailed) { if (await reconcileExit(id, { stop: true })) return true; throw refuse('SDK worker exit cannot be confirmed after restart; inspect its resource-side stop and process evidence') } row.status = 'stopping'; if (row.githubAccess) { row.githubAccess.renewing = false; row.githubAccess.status = 'stopping' }; clearTimeout(control.timer) if (!control.closed) control.process.stdin.end(JSON.stringify({ type: 'stop' }) + '\n') let timer @@ -521,6 +577,7 @@ export function createProjectHarness(rt, { roleGithub } = {}) { } catch { row.scratchCleanupPending = true } // Preserve cleanup debt without inventing an active worker after confirmed exit. } await rt.save() + if (row.exitRecovery?.supported) { try { await exitReceipt(row, 'forget') } catch { /* Retain cleanup debt without losing confirmed exit. */ } } return true } async function chiefTurn({ agent, delegation, definition, context, text, onResult }) { @@ -606,7 +663,7 @@ export function createProjectHarness(rt, { roleGithub } = {}) { return nativeGithub.get(id)?.runtime.env ?? {} } return { enableNativeGithubEnv() { nativeEnvReady = true }, roleGithubActive: id => nativeGithub.has(id), bindNativeGithub, closeNativeGithub, nativeGithubEnv, redactGithub: github.redact, profile, spawn, dispatch, correctInput, checkNativeInput, taskInputWake, release, input, chiefTurn, onResult: fn => { listeners.add(fn); return () => listeners.delete(fn) }, - capabilities: harnessCapabilities, owns: id => Boolean(ownEntry(workers(), id)), hasController: id => active.has(id) && !active.get(id).closed, versions: VERSIONS } + capabilities: harnessCapabilities, owns: id => Boolean(ownEntry(workers(), id)), hasController: id => active.has(id) && !active.get(id).closed, reconcileExit, reconcileExits, versions: VERSIONS } } export function install(rt) { rt.projectHarness = createProjectHarness(rt) diff --git a/core/src/host/project-resources.js b/core/src/host/project-resources.js index 823b45063..b3c539e8f 100644 --- a/core/src/host/project-resources.js +++ b/core/src/host/project-resources.js @@ -275,6 +275,7 @@ export function createProjectResources(rt, { priorExit = priorScopeExited } = {} return changed } async function reconcile() { + await rt.projectHarness?.reconcileExits?.() let changed = false for (const lease of Object.values(leases()).filter(active)) { if (lease.processScope?.hostPid === process.pid && (lease.worker?.kind === 'native' && rt.projectDelivery?.nativeSessions?.currentHost(lease.worker.botId) || lease.worker?.kind === 'sdk' && rt.projectHarness?.hasController(lease.worker.handle) || lease.worker?.kind === 'native-turn' && rt.projectAgents?.hasExchange(lease.worker.handle))) continue diff --git a/core/src/host/project-sdk-exit.mjs b/core/src/host/project-sdk-exit.mjs new file mode 100644 index 000000000..fa0134a6c --- /dev/null +++ b/core/src/host/project-sdk-exit.mjs @@ -0,0 +1,79 @@ +// Resource-side stop acknowledgement plus exact process birth/exit, not Task success. +import { mkdir, lstat, realpath, readFile, writeFile, rename, unlink } from 'node:fs/promises' +import { join, isAbsolute } from 'node:path' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { pathToFileURL } from 'node:url' +import { captureProcessScope, priorHostProcessExited } from './project-process-proof.js' +const exec = promisify(execFile) +export const SDK_EXIT_VERSION = 'project-sdk-exit/v1' +function identity(value) { + if (value?.version !== SDK_EXIT_VERSION || !isAbsolute(value.root ?? '') || !/^[A-Za-z0-9_-]{1,120}$/.test(value.workerId ?? '') || !/^[A-Za-z0-9_-]{1,120}$/.test(value.leaseId ?? '') || !Number.isSafeInteger(value.generation) || value.generation < 1) throw Error('Invalid SDK exit identity') + return { version: SDK_EXIT_VERSION, workerId: value.workerId, leaseId: value.leaseId, generation: value.generation } +} +async function target(value, create = false) { + const expected = identity(value), root = await realpath(value.root), directory = join(root, '.flybot-sdk-exit') + if (create) await mkdir(directory, { mode: 0o700 }).catch(error => { if (error.code !== 'EEXIST') throw error }) + const info = await lstat(directory) + if (!info.isDirectory() || info.isSymbolicLink() || await realpath(directory) !== directory) throw Error('SDK exit directory changed') + return { expected, path: join(directory, expected.workerId + '.json') } +} +async function record(value) { + const { expected, path } = await target(value), info = await lstat(path) + if (!info.isFile() || info.isSymbolicLink() || info.size > 8192) throw Error('SDK exit record changed') + const row = JSON.parse(await readFile(path, 'utf8')) + if (Object.entries(expected).some(([key, field]) => row[key] !== field) || !['running', 'stopped'].includes(row.status)) throw Error('SDK exit record identity changed') + return { row, path } +} +export async function registerSdkExit(value) { + const { expected, path } = await target(value, true) + await writeFile(path, JSON.stringify({ ...expected, scope: await captureProcessScope(), status: 'running' }), { flag: 'wx', mode: 0o600 }) +} +export async function acknowledgeSdkStop(value) { + const { row, path } = await record(value) + if (row.scope.hostPid !== process.pid) throw Error('SDK exit writer changed') + const pending = path + '.stopped' + await writeFile(pending, JSON.stringify({ ...row, status: 'stopped' }), { flag: 'wx', mode: 0o600 }) + await rename(pending, path) +} +// pidfd pins the verified process; no signal fallback to a potentially reused PID. +const stopProcess = `import os,sys,json,signal +s=json.loads(sys.argv[1]) +fd=os.pidfd_open(s['hostPid']) +try: + if open('/proc/sys/kernel/random/boot_id').read().strip()!=s['bootId'] or os.readlink('/proc/self/ns/pid')!=s['pidNamespace']: sys.exit(2) + stat=open('/proc/'+str(s['hostPid'])+'/stat').read(); fields=stat[stat.rfind(')')+2:].split() + if fields[19]!=s['startTicks']: sys.exit(2) + signal.pidfd_send_signal(fd,signal.SIGTERM) +finally: os.close(fd) +` +export async function inspectSdkExit(value, { stop = false } = {}) { + let current + try { current = await record(value) } catch { return { status: 'unknown', reason: 'missing-or-invalid-receipt' } } + if (current.row.status === 'stopped' && await priorHostProcessExited(current.row.scope)) return { status: 'exited', ...identity(value), scope: current.row.scope } + if (!stop) return { status: 'unknown', reason: 'stop-or-process-exit-unconfirmed' } + try { + const scope = current.row.scope + if (!/^[a-f0-9-]{36}$/.test(scope?.bootId ?? '') || !/^\d+$/.test(scope?.startTicks ?? '') || !/^pid:\[\d+\]$/.test(scope?.pidNamespace ?? '') || !Number.isSafeInteger(scope.hostPid) || scope.hostPid < 1) return { status: 'unknown', reason: 'process-birth-unavailable' } + await exec('python3', ['-c', stopProcess, JSON.stringify(scope)], { timeout: 2000, maxBuffer: 8192, env: { PATH: process.env.PATH, LANG: 'C.UTF-8' } }) + } catch { return { status: 'unknown', reason: 'verified-stop-unavailable' } } + const deadline = Date.now() + 6000 + while (Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 100)) + const result = await inspectSdkExit(value) + if (result.status === 'exited') return result + } + return { status: 'unknown', reason: 'stop-or-process-exit-unconfirmed' } +} +export async function forgetSdkExit(value) { + const { path } = await record(value) + await unlink(path) +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const [action, text] = process.argv.slice(2), value = JSON.parse(text) + if (action === 'forget') { await forgetSdkExit(value); process.stdout.write('{}') } + else if (['inspect', 'stop'].includes(action)) process.stdout.write(JSON.stringify(await inspectSdkExit(value, { stop: action === 'stop' }))) + else throw Error('Unsupported SDK exit operation') + } catch { process.stdout.write(JSON.stringify({ status: 'unknown', reason: 'exit-evidence-unavailable' })); process.exitCode = 1 } +} diff --git a/core/test/project-harness.test.js b/core/test/project-harness.test.js index ffbf2bceb..70778a6b0 100644 --- a/core/test/project-harness.test.js +++ b/core/test/project-harness.test.js @@ -3,7 +3,10 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { mkdtemp, mkdir, writeFile, readFile, rm, stat, rename, symlink } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { join, resolve } from 'node:path' +import { createInterface } from 'node:readline' +import { once } from 'node:events' +import { createHash } from 'node:crypto' import { execFileSync } from 'node:child_process' import { createProjectScheduler } from '../src/host/project-scheduler.js' import { createProjectEffects } from '../src/host/project-effects.js' @@ -70,6 +73,77 @@ test('SDK workers reject unregistered Workspaces and missing explicit profiles', await assert.rejects(harness.spawn({ executionProjectId: 'p', executionWorkDir: cwd }), /explicitly authorized/) }) +for (const priorState of ['unreachable', 'released-worker', 'released-lease', 'stale-generation', 'legacy-no-proof']) test('controller-lost SDK release reconciles ' + priorState + ' using exact exit evidence', async t => { + const { rt, cwd } = await fixture(t) + const scope = JSON.parse(execFileSync(process.execPath, ['--input-type=module', '-e', `import {captureProcessScope} from ${JSON.stringify(new URL('../src/host/project-process-proof.js', import.meta.url).href)};console.log(JSON.stringify(await captureProcessScope()))`], { encoding: 'utf8', env: { PATH: process.env.PATH } })) + const id = 'sdk-controller-lost', root = rt.config.projectResources.r.root + for (const name of ['project-sdk-exit.mjs', 'project-process-proof.js']) await writeFile(join(rt.config.projectResources.r.runtimeDir, 'src/host', name), await readFile(new URL('../src/host/' + name, import.meta.url))) + await mkdir(join(root, '.flybot-sdk-exit')) + await writeFile(join(root, '.flybot-sdk-exit', id + '.json'), JSON.stringify({ version: 'project-sdk-exit/v1', workerId: id, leaseId: 'l', generation: 1, scope, status: 'stopped' })) + rt.state.projectHarnessWorkers = { [id]: { id, projectId: 'p', attemptId: 'a', role: 'developer', resourceId: 'r', leaseId: 'l', generation: 1, status: priorState === 'released-worker' ? 'released' : 'unreachable', cwd, exitRecovery: { version: 'project-sdk-exit/v1' } } } + if (priorState === 'released-worker') rt.state.projectHarnessWorkers[id].releaseProof = { kind: 'sdk-stop-ack-and-process-exit', workerId: id, generation: 1, at: Date.now() } + Object.assign(rt.state.projectResourceLeases.l, { status: priorState === 'released-lease' ? 'released' : 'unreachable', worker: { kind: 'sdk', handle: id } }) + const restored = createProjectHarness(rt) + if (priorState === 'legacy-no-proof') { + delete rt.state.projectHarnessWorkers[id].exitRecovery + await rm(join(root, '.flybot-sdk-exit', id + '.json')) + await assert.rejects(restored.release(id), /exit cannot be confirmed/) + assert.equal(rt.state.projectResourceLeases.l.status, 'unreachable') + delete rt.state.projectHarnessWorkers[id] + return + } + if (priorState === 'stale-generation') { + const command = rt.projectResources.command + rt.projectResources.command = async (...args) => { const result = await command(...args); rt.state.projectResourceLeases.l.generation = 2; return result } + await assert.rejects(restored.release(id), /exit cannot be confirmed/) + assert.equal(rt.state.projectResourceLeases.l.status, 'unreachable') + delete rt.state.projectHarnessWorkers[id] // No live process exists in this synthetic stale-binding fixture. + return + } + assert.equal(await restored.release(id), true) + assert.equal(rt.state.projectResourceLeases.l.status, 'released') + assert.equal(rt.state.projectHarnessWorkers[id].releaseProof.kind, 'sdk-stop-ack-and-process-exit') +}) + +for (const recovery of ['release', 'reconcile']) test('controller-lost host ' + recovery + ' waits for the actual resource worker to settle and exit', { timeout: 15000 }, async t => { + const { rt, cwd } = await fixture(t), id = 'sdk-live-controller-lost' + rt.config.projectResources.r.runtimeDir = resolve(import.meta.dirname, '..') + rt.state.projectHarnessWorkers = { [id]: { id, projectId: 'p', attemptId: 'a', role: 'developer', resourceId: 'r', leaseId: 'l', generation: 1, status: 'unreachable', cwd, exitRecovery: { version: 'project-sdk-exit/v1' } } } + await rt.projectResources.markStarted('l', 1, { kind: 'sdk', handle: id }) + const identity = { version: 'project-sdk-exit/v1', root: rt.config.projectResources.r.root, workerId: id, leaseId: 'l', generation: 1 } + const child = rt.projectResources.startProcess('p', 'r', { cwd, argv: ['node', resolve(import.meta.dirname, '../src/host/project-harness-worker.mjs'), JSON.stringify(identity)] }) + child.stderr.resume(); child.stdin.on('error', () => {}) + const exit = once(child, 'close'), lines = createInterface({ input: child.stdout }) + t.after(async () => { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); await exit; lines.close() }) + await once(lines, 'line') + rt.projectHarness = createProjectHarness(rt) + assert.equal(rt.projectResources.occupied('r'), 1) + if (recovery === 'release') assert.equal(await rt.projectHarness.release(id), true) + else { child.stdin.end(); await exit; await rt.projectResources.reconcile() } + await exit + assert.equal(rt.projectResources.occupied('r'), 0) + assert.equal(rt.state.projectHarnessWorkers[id].releaseProof.source, 'resource-stopped-receipt') + assert.equal(rt.state.tasks.t.status, undefined, 'Physical exit does not complete the Task') +}) + +for (const previousStatus of ['unreachable', 'released']) test('legacy ' + previousStatus + ' SDK reconciles only its saved stop event and zero process exit', async t => { + const { rt, harness, cwd } = await fixture(t) + const start = rt.projectResources.startProcess; let child + rt.projectResources.startProcess = (...args) => (child = start(...args)) + const id = await harness.spawn({ executionProjectId: 'p', executionWorkDir: cwd, prompt: 'Own legacy worker' }) + const exited = once(child, 'close') + child.stdin.end(JSON.stringify({ type: 'stop' }) + '\n'); await exited + const row = rt.state.projectHarnessWorkers[id] + const stoppedHash = createHash('sha256').update(JSON.stringify({ type: 'stopped', released: true })).digest('hex') + for (let n = 0; n < 100 && row.lastEventHash !== stoppedHash; ++n) await new Promise(resolve => setTimeout(resolve, 5)) + assert.equal(row.lastEventHash, stoppedHash) + delete row.exitRecovery; row.status = previousStatus; rt.state.projectResourceLeases.l.status = previousStatus + rt.projectHarness = createProjectHarness(rt) + await rt.projectResources.reconcile() + assert.equal(row.status, 'released'); assert.equal(rt.projectResources.occupied('r'), 0) + assert.equal(row.releaseProof.source, 'durable-host-stop-event') +}) + test('non-App Dev receives Git mutation rights only for its bound isolated resource clone', async t => { const { harness, rt, cwd } = await fixture(t) execFileSync('git', ['init', '-q', cwd]) diff --git a/core/test/project-sdk-exit-sdk.test.js b/core/test/project-sdk-exit-sdk.test.js new file mode 100644 index 000000000..7bfcc2c4d --- /dev/null +++ b/core/test/project-sdk-exit-sdk.test.js @@ -0,0 +1,44 @@ +// Opt-in actual SDK/CLI abort and worker exit; provider is a held local fixture. +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { createServer } from 'node:http' +import { spawn, execFileSync } from 'node:child_process' +import { once } from 'node:events' +import { createInterface } from 'node:readline' +import { mkdtemp, mkdir, rm, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { inspectSdkExit, SDK_EXIT_VERSION } from '../src/host/project-sdk-exit.mjs' +if (process.env.FLYBOT_TEST_EXIT_SDK === '1') test('actual Codex SDK settles after controller pipe loss and leaves verifiable resource exit', { timeout: 45000 }, async () => { + const root = await mkdtemp(join(tmpdir(), 'flybot-sdk-exit-actual-')), cwd = join(root, 'work'), home = join(root, 'home') + await mkdir(cwd); await mkdir(home) + execFileSync('git', ['init', '-q', cwd]) + let reached + const called = new Promise(resolve => { reached = resolve }) + const server = createServer((request, response) => { request.resume(); reached(); response.writeHead(200, { 'content-type': 'text/event-stream' }); response.write(': hold until SDK abort\n\n') }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const identity = { version: SDK_EXIT_VERSION, root, workerId: 'actual-sdk-worker', leaseId: 'actual-sdk-lease', generation: 1 } + const child = spawn(process.execPath, [resolve(import.meta.dirname, '../src/host/project-harness-worker.mjs'), JSON.stringify(identity)], { env: { PATH: process.env.PATH }, stdio: ['pipe', 'pipe', 'pipe'] }) + const exit = once(child, 'close'), lines = createInterface({ input: child.stdout }) + child.stderr.resume(); child.stdin.on('error', () => {}) + try { + const [first] = await once(lines, 'line'), ready = JSON.parse(first) + assert.equal(ready.sdk.codex, '0.166.0') + child.stdin.write(JSON.stringify({ type: 'start', settings: { harness: 'codex', cwd, model: 'fixture-model', noTools: true, env: { HOME: home, CODEX_HOME: home, PATH: process.env.PATH }, config: { model_provider: 'fixture', model_providers: { fixture: { name: 'fixture', base_url: `http://127.0.0.1:${server.address().port}`, wire_api: 'responses', requires_openai_auth: false } } } }, prompt: 'Wait for this isolated fixture response' }) + '\n') + await Promise.race([called, new Promise((_, reject) => setTimeout(() => reject(Error('Actual SDK did not reach fixture provider')), 20000).unref())]) + const cliPids = (await readFile(`/proc/${child.pid}/task/${child.pid}/children`, 'utf8')).trim().split(/\s+/).filter(Boolean) + assert.ok(cliPids.length, 'Actual SDK spawned its CLI') + assert.equal((await inspectSdkExit(identity)).status, 'unknown') + child.stdout.destroy(); child.stdin.end() + const [code] = await exit + assert.equal(code, 0) + assert.equal((await inspectSdkExit(identity)).status, 'exited', 'SDK abort must settle before the actual worker acknowledges stop') + for (const pid of cliPids) { + try { const stat = await readFile(`/proc/${pid}/stat`, 'utf8'); assert.ok(['Z', 'X'].includes(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0]), 'SDK CLI must also have exited') } + catch (error) { if (error.code !== 'ENOENT') throw error } + } + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + await exit; lines.close(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); await rm(root, { recursive: true, force: true }) + } +}) diff --git a/core/test/project-sdk-exit.test.js b/core/test/project-sdk-exit.test.js new file mode 100644 index 000000000..add8de59a --- /dev/null +++ b/core/test/project-sdk-exit.test.js @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { spawn, execFileSync } from 'node:child_process' +import { createInterface } from 'node:readline' +import { once } from 'node:events' +import { mkdtemp, mkdir, rm, readFile, writeFile, symlink, unlink } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { SDK_EXIT_VERSION, registerSdkExit, acknowledgeSdkStop, inspectSdkExit, forgetSdkExit } from '../src/host/project-sdk-exit.mjs' +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), 'flybot-sdk-exit-')) + t.after(() => rm(root, { recursive: true, force: true })) + const value = { version: SDK_EXIT_VERSION, root, workerId: 'sdk-exit-fixture', leaseId: 'lease-fixture', generation: 1 } + return { value, path: join(root, '.flybot-sdk-exit', value.workerId + '.json') } +} +async function worker(t) { + const f = await fixture(t) + const child = spawn(process.execPath, [fileURLToPath(new URL('../src/host/project-harness-worker.mjs', import.meta.url)), JSON.stringify(f.value)], { env: { PATH: process.env.PATH }, stdio: ['pipe', 'pipe', 'pipe'] }) + child.stderr.resume(); child.stdin.on('error', () => {}) + const exit = once(child, 'close'), lines = createInterface({ input: child.stdout }) + t.after(async () => { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); await exit; lines.close() }) + const [line] = await Promise.race([once(lines, 'line'), exit.then(() => { throw Error('Worker did not register') })]) + assert.equal(JSON.parse(line).exitRecovery, SDK_EXIT_VERSION) + return { ...f, child, exit } +} +test('invalid/missing SDK exit evidence never implies release and registration cannot overwrite another record', async t => { + const f = await fixture(t) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + await assert.rejects(registerSdkExit({ ...f.value, workerId: '../foreign' }), /identity/) + await registerSdkExit(f.value) + await assert.rejects(registerSdkExit(f.value), /EEXIST/) + assert.equal((await inspectSdkExit({ ...f.value, generation: 2 }, { stop: true })).status, 'unknown') + await acknowledgeSdkStop(f.value) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown', 'A stopped receipt alone cannot prove the writer process exited') + await writeFile(f.path, 'malformed receipt') + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + await rm(f.path); await symlink(join(f.value.root, 'private'), f.path) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + await unlink(f.path); await writeFile(f.path, 'x'.repeat(9000)) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') +}) +test('SDK exit records refuse a symlinked supervisor directory', async t => { + const f = await fixture(t), foreign = join(f.value.root, 'foreign') + await mkdir(foreign); await symlink(foreign, join(f.value.root, '.flybot-sdk-exit')) + await assert.rejects(registerSdkExit(f.value), /directory changed/) +}) +test('resource exit CLI rejects unsupported operations with a bounded machine-readable result', () => { + assert.throws(() => execFileSync(process.execPath, [fileURLToPath(new URL('../src/host/project-sdk-exit.mjs', import.meta.url)), 'unsupported', '{}'], { encoding: 'utf8', stdio: 'pipe' }), error => { + assert.deepEqual(JSON.parse(error.stdout), { status: 'unknown', reason: 'exit-evidence-unavailable' }); return true + }) +}) +if (process.platform === 'linux') { + test('actual idle SDK worker can be stopped by exact resource process identity after controller loss', { timeout: 15000 }, async t => { + const f = await worker(t) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + assert.equal((await inspectSdkExit(f.value, { stop: true })).status, 'exited') + await f.exit + assert.equal(JSON.parse(await readFile(f.path, 'utf8')).status, 'stopped') + await forgetSdkExit(f.value) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + }) + test('closed controller pipes still produce a durable settled stop receipt', { timeout: 10000 }, async t => { + const f = await worker(t) + f.child.stdout.destroy(); f.child.stdin.end() + await f.exit + assert.equal((await inspectSdkExit(f.value)).status, 'exited') + }) + test('a killed worker without settled stop acknowledgement remains unknown', { timeout: 10000 }, async t => { + const f = await worker(t) + f.child.kill('SIGKILL'); await f.exit + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + }) + test('a process that cannot acknowledge settled shutdown retains unknown exit after the stop budget', { timeout: 10000 }, async t => { + const f = await fixture(t) + const module = new URL('../src/host/project-sdk-exit.mjs', import.meta.url).href + const child = spawn(process.execPath, ['--input-type=module', '-e', `import {registerSdkExit} from ${JSON.stringify(module)};process.on('SIGTERM',()=>{});await registerSdkExit(${JSON.stringify(f.value)});console.log('ready');setInterval(()=>{},1000)`], { env: { PATH: process.env.PATH }, stdio: ['pipe', 'pipe', 'pipe'] }) + const exit = once(child, 'close'), lines = createInterface({ input: child.stdout }); child.stderr.resume() + t.after(async () => { child.kill('SIGKILL'); await exit; lines.close() }) + await once(lines, 'line') + assert.equal((await inspectSdkExit(f.value, { stop: true })).status, 'unknown') + assert.equal(child.exitCode, null) + assert.equal(JSON.parse(await readFile(f.path, 'utf8')).status, 'running') + }) + test('a mismatched birth cannot signal a reused PID or release another generation', { timeout: 15000 }, async t => { + const f = await worker(t), original = await readFile(f.path, 'utf8'), row = JSON.parse(original) + row.scope.startTicks = String(Number(row.scope.startTicks) + 1) + await writeFile(f.path, JSON.stringify(row)) + assert.equal((await inspectSdkExit(f.value, { stop: true })).status, 'unknown') + assert.equal(f.child.exitCode, null) + await writeFile(f.path, original) + assert.equal((await inspectSdkExit(f.value, { stop: true })).status, 'exited') + await f.exit + }) +} diff --git a/decisions/2026-11-05-controller-loss-exit-evidence.md b/decisions/2026-11-05-controller-loss-exit-evidence.md new file mode 100644 index 000000000..aa11e05f1 --- /dev/null +++ b/decisions/2026-11-05-controller-loss-exit-evidence.md @@ -0,0 +1,29 @@ +# Reconcile SDK exit after losing the in-memory controller + +Date: 2026-11-05. Status: accepted by the Owner's explicit continuation of #2483, +targeting delivery/ai-centric-development-service in this foreground session. + +The original release required a process-local controller even after a resource worker +had stopped. Reproduce the failure at7e79762d7 and reuse the original SDK stop/exit +contract with a small resource-side receipt. Bind it to worker/lease/generation and +actual process birth; store outside model-write roots. A settled acknowledgement plus +verified process exit is physical evidence, not code correctness or Task acceptance. +Explicit release can request controlled SIGTERM through a verified pidfd. Background +reconciliation observes only; it chooses no role or escalation. No new coordinator, +timer or Effect kind is introduced. + +Existing trusted host stop/exit records can repair historical partial releases. Old +records lacking actual proof remain unknown; no metadata migration fabricates exit. +The selected resource runtime needs the receipt-aware worker for future recovery. +Linux/Python pidfd support is needed for a lost-controller stop request; ordinary +connected-controller behavior and conservative unsupported-platform handling remain. + +Unit/integration tests cover actual worker transport and process exit, generation/PID +mismatch, missing acknowledgement and partial release states. Opt-in actual pinned +Codex SDK/CLI tests hold a local fixture provider, disconnect controller pipes and +verify SDK/worker exit on host and isolated Docker. These incur no paid inference and +do not constitute live OpenToday/private App acceptance or deployment. + +Verification caught accumulated policy text exceeding its existing budget. Preserve +earlier PROJECT chronology verbatim in history, retain current constraints inline +and keep every mandatory source and the384000-byte cap unchanged. diff --git a/docs/backend-operations.md b/docs/backend-operations.md index 5a5b47a7d..431b1a607 100644 --- a/docs/backend-operations.md +++ b/docs/backend-operations.md @@ -246,6 +246,15 @@ Use GET /ready to detect read-only/unavailable backend state; GET /health only p ### Queued without a worker, or pending Chief inputs (#2483) +For controller-lost SDK work, current resource workers preserve a settled stop receipt +outside model-write roots. The existing reconciliation checks its exact identity and +resource process exit, then aligns worker and lease. Explicit stop uses birth-verified +Linux pidfd through Python3; it cannot signal a reused PID. A saved trusted host stop +event/exit can repair legacy partial releases. Missing acknowledgement, missing birth +identity or a forcibly killed worker remains unknown. Upgrade the selected resource's +FlyBot runtime as well as the host to emit new receipts. Do not treat container Up, +occupied=0, a missing controller or PID disappearance as sufficient exit proof. + After restart, current blocked/interrupted role work now notifies the configured Chief through its existing controller. `chief.state` Run.source identifies the interrupted claim, policy and allocation fingerprint; repeated scheduler passes diff --git a/docs/history/2026-11-05-controller-exit-memory-checkpoints.md b/docs/history/2026-11-05-controller-exit-memory-checkpoints.md new file mode 100644 index 000000000..68cf38082 --- /dev/null +++ b/docs/history/2026-11-05-controller-exit-memory-checkpoints.md @@ -0,0 +1,45 @@ +# Earlier runtime and recovery checkpoints + +Preserved verbatim from PROJECT.md; relative links retain their original repository-root context. Historical progress grants no new authority. + +## SDK authentication — 2026-11-04 + +#2398 binds readiness and setup to the existing resource profile, preserving its +paths and Task-role context. Known SDK authentication failures reach authRequired +without becoming code verdicts. No OAuth broker, credential copying or live login. + +## Role App delivery — 2026-11-04 + +#2365 opts Project roles into explicit App connections and dedicated short-lived +credential helpers. Original candidate Effects observe SDK commits/PRs without +resealing; actual App Review and host merge/Chief acceptance stay separate. +No Owner fallback or live migration. [Decision](decisions/2026-11-04-role-github-apps.md) +and [operator contract](docs/role-github-apps.md); live App acceptance requires authorized installations. + +## Follow-up reliability delivery — 2026-11-04 + +Owner authorizes sequential #2336/#2337/#2334/#2346/#2338/#2347/#2331/#2342 +implementation, one PR each, tests/memory/docs and validated merge to the delivery +branch. #2336 establishes owned reservation rollback and a durable pre-spawn fence; +unknown execution, workspace/context and current authority remain protected. +#2337 validates the actual SDK-resolved CLI and role profile before inference; +configuration readiness is separate from model/test/exit evidence. +#2334 separates executor dependency readiness from host test outcomes and preserves +same-candidate retry evidence; remote unknown exit retains preparation occupancy. +#2346 admits cwd-only Task-role restoration with current binding/exit proof and +explicit resumed/reconstructed identity; clearing bindings never resets context. +No live task/configuration/credential mutation or deployment is implied. + +## Execution recovery — 2026-11-04 + +#2325 aligns actual reused SDK/resource checkouts with pinned baseline/seed facts, +preserves drafts and admits explicit inspect/integrate inputs through existing +Effects. Chief selects only approved repository sources. #2327 adds bounded, durable Chief protocol correction in the resident Session; +exhaustion retains incomplete judgement without automatic escalation. Follow-ups: +#2328 adds confirmed-idle wait suspension and FIFO continuation re-admission +through the original queue; exit-unknown capacity remains held. #2330 adds durable scheduled inspection reconciliation, configured deadlines, +exit-proven bounded recovery and health visibility; independent Task refusals do not +abort other decisions. No automatic business escalation or live rollout. See +[delivery contract](docs/subsystems/project-delivery.md). Live Task mutation and +deployment remain separate from source/test delivery. + diff --git a/docs/openapi/flybot.openapi.json b/docs/openapi/flybot.openapi.json index 21f014a59..ce4f0d5a0 100644 --- a/docs/openapi/flybot.openapi.json +++ b/docs/openapi/flybot.openapi.json @@ -38030,7 +38030,7 @@ "notes": "payload.requestId identifies the original SDK permission question; outer requestId is the independent command receipt ID. A role string never grants execution identity.", "implementation": { "path": "core/src/host/project-harness.js", - "sha256": "2010cf0edef08137cf55d3abffdd7780764219824aec1b76cbc8661a8df39dd9" + "sha256": "fddda86a073dab3e2de4ee517f190841b3a1f339e895b5050ecf175e27ee6551" } }, { @@ -43350,7 +43350,7 @@ "notes": "All current authorization, Project/Task bindings and domain fences are rechecked server-side.", "implementation": { "path": "core/src/host/project-resources.js", - "sha256": "9a267001af697abdcdd9d3dd3e06e97d9e3042b77e6e31cb781ca57a4ada5920" + "sha256": "49a680e91c703f514ff1b8d80aa77ab07860dcc61c81d6ae80fb8e62fd9d24d4" } }, { @@ -48993,7 +48993,7 @@ "notes": "payload.requestId identifies the original SDK permission question; outer requestId is the independent command receipt ID. A role string never grants execution identity.", "implementation": { "path": "core/src/host/project-harness.js", - "sha256": "2010cf0edef08137cf55d3abffdd7780764219824aec1b76cbc8661a8df39dd9" + "sha256": "fddda86a073dab3e2de4ee517f190841b3a1f339e895b5050ecf175e27ee6551" }, "aliasOf": "development.command.execution.input" }, @@ -49527,7 +49527,7 @@ "core/src/host/project-github.js": "29c81ebad0d3cb094bb8c549f8ee89c674cb1685c2f73f79095b53a121a97c52", "core/src/host/project-harness-auth.js": "2684b6687433e128197a0cc67b671bbd02c3c71e39428af7393d0c0ca1658ac7", "core/src/host/project-harness-turn.js": "7a08f8d0e9c7ac6812a55714bde9505640f245783aa28dde3a6d9a2de161c6e6", - "core/src/host/project-harness.js": "2010cf0edef08137cf55d3abffdd7780764219824aec1b76cbc8661a8df39dd9", + "core/src/host/project-harness.js": "fddda86a073dab3e2de4ee517f190841b3a1f339e895b5050ecf175e27ee6551", "core/src/host/project-history-pages.js": "38958623f0aa98099e570559c55a207285fd0349689ffdcec47b965c40f4e1bc", "core/src/host/project-inputs.js": "dc64abe3e3af9b0456cc75f83ae8a22fd1ff587248a33d0b4df4416082eeccf3", "core/src/host/project-lifecycle.js": "055784e0a1cbf2a9e465e990c60fbd6df4b8844052db43bbcb7cae5de9282936", @@ -49543,7 +49543,7 @@ "core/src/host/project-releases.js": "ac588064b036f2273c7b61cf3389c1da720e6f2166c37d269afd9d9095eb6885", "core/src/host/project-repository-policy.js": "c16f469f3275f61e817bcad0bc8c753f01b034db6562c4e3d197f0349174823a", "core/src/host/project-resource-workspaces.js": "cc9525aab5ea9ea09d4fb2c557a1048fa034d638c0b80dbebea9cc8eba0ede8b", - "core/src/host/project-resources.js": "9a267001af697abdcdd9d3dd3e06e97d9e3042b77e6e31cb781ca57a4ada5920", + "core/src/host/project-resources.js": "49a680e91c703f514ff1b8d80aa77ab07860dcc61c81d6ae80fb8e62fd9d24d4", "core/src/host/project-review-evidence.js": "da3f72d2cefcf725c19f590e926ce391af36d45df97163b0d4a07699cef5ceb9", "core/src/host/project-reviewed-merge.js": "f178310efb224fa9bd51042e4a1b14122fc4fd9a155a7e43da17a5e668a29aa8", "core/src/host/project-role-admission.js": "ccd5f3ac4a5d1f0010a3e533e6809b20a4f88551fc55e6c775d8289d8f9e100f", diff --git a/docs/subsystems/backend-service.md b/docs/subsystems/backend-service.md index 4d2a21415..7b47cba7e 100644 --- a/docs/subsystems/backend-service.md +++ b/docs/subsystems/backend-service.md @@ -30,6 +30,10 @@ source fingerprints reflect the host adapter fix. See [Harness](project-harness. # Persistent source backend entry (#2189) +#2483 controller-loss reconciliation strengthens execution evidence behind the same +release/reconcile interfaces. No new public operation or privilege; OpenAPI source +fingerprints reflect the Harness/resource implementation changes. + #2483 interrupted-role facts now reach existing Chief inspection without a manual API call. `chief.state` source identifies the claim/policy/allocation fingerprint; TaskView reflects the applied recovery, not notification success. Public operations diff --git a/docs/subsystems/project-harness.md b/docs/subsystems/project-harness.md index f24f14f17..bf8ddaf02 100644 --- a/docs/subsystems/project-harness.md +++ b/docs/subsystems/project-harness.md @@ -28,10 +28,33 @@ owned_paths: - core/test/project-harness.test.js - core/test/project-harness-turn.test.js - core/test/project-harness-git-sdk.test.js + - core/src/host/project-sdk-exit.mjs + - core/test/project-sdk-exit.test.js + - core/test/project-sdk-exit-sdk.test.js --- # Harness adapters (fork M7.3) +## Controller-loss exit evidence (#2483) + +New resource workers register their worker/lease/generation and actual process birth +in a private `.flybot-sdk-exit` artifact outside SDK workspace/scratch write grants. +Only after abort/permissions/credential cleanup and settled SDK work do they atomically +write stopped. EOF, closed output and SIGTERM use that same shutdown path. A missing +controller can inspect the matching receipt plus actual resource-side process exit; +neither receipt alone nor a missing PID grants release. Explicit release may signal +the birth-verified process using Linux pidfd (Python3); there is no PID-only kill fallback. +Unsupported identity/stop facilities retain unknown exit. + +The existing resource reconciliation pass also repairs partial worker/lease release. +Trusted saved SDK stop/exit proof or hash-matched host stopped event plus saved zero +exit can reconcile legacy records, including occupied=0 with a stale worker record. +Without those facts, old unregistered workers remain unknown; evidence is not invented. +Reuse `sdk-stop-ack-and-process-exit`, original Session release, current binding/generation +checks and scheduler. No new service, Task store, business workflow or automatic next +role is added. Confirmed proof persists before receipt cleanup; cleanup debt does not +reoccupy a released resource. Task results/external writes still need original reconciliation. + ## Isolated resource Git capability (#2483) An active Developer's registered `resource-clone` receives write access to its own diff --git a/docs/subsystems/project-resources.md b/docs/subsystems/project-resources.md index db8c5ecce..18cc2f0b0 100644 --- a/docs/subsystems/project-resources.md +++ b/docs/subsystems/project-resources.md @@ -56,6 +56,12 @@ than inferred from configured slot counts. Host process metrics retain their own ## Admission and lifecycle +#2483 runs the original Harness exit reconciliation in this existing pass, before +resource admission. Exact SDK stop/process evidence reconciles both worker and lease, +including partially saved releases. Current controllers retain their original stop path; +unknown or mismatched generations retain occupancy. Resource-side inspection uses only +the currently bound resource/runtime/UID; no credential login or resource migration. + Role execution configuration has candidate `resourceIds`, Harness and optional positive `maxConcurrent`. Choose the least occupied compatible ready environment and retain the single-resource binding for the Session. There is no broadcast or in-flight migration. diff --git a/patches/ds-bot/files/src/host/project-harness-worker.mjs b/patches/ds-bot/files/src/host/project-harness-worker.mjs index de837d66b..2f6265e6f 100644 --- a/patches/ds-bot/files/src/host/project-harness-worker.mjs +++ b/patches/ds-bot/files/src/host/project-harness-worker.mjs @@ -6,9 +6,14 @@ import { readFileSync, existsSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { runHarnessTurn } from './project-harness-turn.js' +import { registerSdkExit, acknowledgeSdkStop, SDK_EXIT_VERSION } from './project-sdk-exit.mjs' +const exitIdentity = process.argv[2] ? JSON.parse(process.argv[2]) : null +if (exitIdentity) await registerSdkExit(exitIdentity) const input = createInterface({ input: process.stdin, crlfDelay: Infinity }) let githubRuntime -const write = value => process.stdout.write(JSON.stringify(githubRuntime ? githubRuntime.redact(value) : value) + '\n') +let outputClosed = false +const write = value => { if (!outputClosed) process.stdout.write(JSON.stringify(githubRuntime ? githubRuntime.redact(value) : value) + '\n') } +process.stdout.on('error', () => { outputClosed = true; void shutdown() }) const answers = new Map(), credentials = new Map() let controller, sessionId, settings, busy = false, shuttingDown = false const completedTurns = new Map() @@ -16,6 +21,7 @@ const permission = request => new Promise(resolve => { const id = randomUUID(); answers.set(id, resolve); write({ type: 'permission', id, request }) }) async function turn(message) { + if (shuttingDown) return if (message.turnId && completedTurns.has(message.turnId)) { write(completedTurns.get(message.turnId)); return } if (busy) { write({ type: 'failure', reason: 'SDK Session is already answering' }); return } busy = true; controller = new AbortController() @@ -58,10 +64,12 @@ async function shutdown() { for (const resolve of answers.values()) resolve({ allow: false }) for (let n = 0; busy && n < 100; n++) await new Promise(resolve => setTimeout(resolve, 50)) if (busy) return // No exit claim while the SDK has not settled; host retains capacity. + if (exitIdentity) await acknowledgeSdkStop(exitIdentity) write({ type: 'stopped', released: true }) input.close(); process.stdin.pause(); setTimeout(() => process.exit(0), 50) } input.on('close', () => { void shutdown() }) +process.on('SIGTERM', () => { void shutdown() }) function sdkVersion(name) { try { let directory = dirname(fileURLToPath(import.meta.resolve(name))) @@ -73,4 +81,4 @@ function sdkVersion(name) { } catch { /* Missing SDK is an explicit failed handshake, not a simulated version. */ } return null } -write({ type: 'ready', roleGithub: 'role-github/v1', receiptCorrection: 'input-correction/v1', pid: process.pid, sdk: { codex: sdkVersion('@openai/codex-sdk'), 'claude-agent': sdkVersion('@anthropic-ai/claude-agent-sdk') } }) +write({ type: 'ready', roleGithub: 'role-github/v1', receiptCorrection: 'input-correction/v1', exitRecovery: exitIdentity ? SDK_EXIT_VERSION : null, pid: process.pid, sdk: { codex: sdkVersion('@openai/codex-sdk'), 'claude-agent': sdkVersion('@anthropic-ai/claude-agent-sdk') } }) diff --git a/patches/ds-bot/files/src/host/project-harness.js b/patches/ds-bot/files/src/host/project-harness.js index edaf68d6e..842eaf125 100644 --- a/patches/ds-bot/files/src/host/project-harness.js +++ b/patches/ds-bot/files/src/host/project-harness.js @@ -5,7 +5,7 @@ import { roleGithubRuntime } from './project-role-github-runtime.mjs' // Harness worker lifecycle plugs into the existing Workflow. It never accepts a Task. import { randomUUID } from 'node:crypto' import { createHash } from 'node:crypto' -import { mkdir, appendFile, readFile, copyFile, writeFile } from 'node:fs/promises' +import { mkdir, appendFile, readFile, copyFile, writeFile, realpath, lstat } from 'node:fs/promises' import { join, resolve, relative, dirname } from 'node:path' import { createInterface } from 'node:readline' import { verifyRolePolicy, roleExecutionPrompt } from './project-role-policy.js' @@ -13,10 +13,11 @@ import { sdkContinuationArchive } from './project-session-continuation.js' import { fileDigest } from './project-resource-workspaces.js' import { copyDependencyCacheScript, executionOutcome } from './project-execution-environment.js' import { gitWritableRoots } from './project-git-sandbox.js' +import { SDK_EXIT_VERSION } from './project-sdk-exit.mjs' import { ownEntry, setEntry } from './store.js' import { checkContextPrompt, resolveContextBudget } from './project-context-budget.js' import { requireExecutionAuthority } from './project-execution-authority.js' -import { taskSession, backendBindingKey, backendContextKey, resumableBinding, roleDefinitionKey } from './project-task-sessions.js' +import { taskSession, backendBindingKey, backendContextKey, resumableBinding, roleDefinitionKey, confirmedWorkerExit } from './project-task-sessions.js' const refuse = message => Object.assign(new Error(message), { code: 'bot/harness' }) const sha = text => createHash('sha256').update(text).digest('hex') const VERSIONS = { codex: '0.166.0', 'claude-agent': '0.3.316' } @@ -151,7 +152,7 @@ export function createProjectHarness(rt, { roleGithub } = {}) { const row = { id, ...(conversation?.roomId ? { roomId: conversation.roomId } : {}), model: session.execution.model ?? selected.selected.model ?? null, chiefRunId: chief?.runId ?? null, chiefInvocation: chief?.runId ? rt.state.projectExecutions?.[chief.runId]?.invocation ?? null : null, visibility: conversation?.visibility ?? 'project', ...(conversation?.visibility === 'private' ? { memberId: conversation.memberId } : {}), projectId: session.projectId, taskId: session.taskId, attemptId: session.attemptId, role: session.role, policyRole: session.policyRole ?? session.definition?.policyRole ?? session.role, sessionKey: session.id, resourceId: session.resourceLease.resourceId, leaseId: session.resourceLease.id, generation: session.resourceLease.generation, harness: session.execution.harness, sdkVersion: null, expectedSdkVersion: VERSIONS[session.execution.harness], principalId: selected.principalId, profile: selected.profile, - cwd: workspace.path, scratchDirectory, dependencyPreparationId: session.environmentPreparation?.id ?? null, spawnPhase: 'registered', status: 'prepared', executionStartedAt: null, createdAt: Date.now(), eventCount: 0 } + cwd: workspace.path, scratchDirectory, exitRecovery: { version: SDK_EXIT_VERSION }, dependencyPreparationId: session.environmentPreparation?.id ?? null, spawnPhase: 'registered', status: 'prepared', executionStartedAt: null, createdAt: Date.now(), eventCount: 0 } if (logical) Object.assign(row, { logicalSessionId: logical.id, bindingKey, bindingVersion: 'task-role-binding/v3', contextIdentityKey: backendContextKey({ ...identity, definition: definitionKey }), definitionKey, ...(previous ? { ...(previous.restoration?.mode === 'reconstructed' ? {} : { backendSessionId: previous.sessionId }), ...(previous.restoration ? { restoration: previous.restoration } : {}), continuedFrom: previous.workerId, ...(previous.key !== bindingKey ? { migratedBindingKey: previous.key } : {}) } : { rotationReason: 'First Task-role execution' }) }) else if (chief) Object.assign(row, { logicalSessionId: chief.agentId, bindingKey, ...(chief.resumeSessionId ? { backendSessionId: chief.resumeSessionId, continuedFrom: chief.previousWorkerId } : {}) }) setEntry(workers(), id, row) @@ -185,7 +186,8 @@ export function createProjectHarness(rt, { roleGithub } = {}) { if (row.restoration) session.restoration = structuredClone(row.restoration) } await rt.projectResources.fenceSpawn(row.leaseId, row.generation, id) - const process = rt.projectResources.startProcess(row.projectId, row.resourceId, { argv: ['node', workerPath], cwd: workspace.path }) + const exitIdentity = { version: SDK_EXIT_VERSION, root: selected.resource.root, workerId: id, leaseId: row.leaseId, generation: row.generation } + const process = rt.projectResources.startProcess(row.projectId, row.resourceId, { argv: ['node', workerPath, JSON.stringify(exitIdentity)], cwd: workspace.path }) const control = { process, chief, githubScope, contextBudget, resumeSessionId: chief?.resumeSessionId ?? (previous?.restoration?.mode === 'reconstructed' ? undefined : previous?.sessionId), continuedPolicyHash: previous?.policySha256, archivePath, continuationPath, logical, bindingKey, env: selected.env, profile: { ...selected.selected, ...(session.execution.model ? { model: session.execution.model } : {}) }, session, prompt: spec.prompt, outputs: [], closed: false, ready: false } row.spawnPhase = 'spawned' active.set(id, control) @@ -204,6 +206,7 @@ export function createProjectHarness(rt, { roleGithub } = {}) { try { const signal = JSON.parse(line), lease = ownEntry(rt.state.projectResourceLeases ?? {}, row.leaseId) if (signal.type === 'ready' && active.get(id) === control) { + row.exitRecovery.supported = signal.exitRecovery === SDK_EXIT_VERSION control.receiptCorrectionSupported = signal.receiptCorrection === 'input-correction/v1' control.readyObserved = true control.observedSdkVersion = /^\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?$/.test(signal.sdk?.[row.harness] ?? '') ? signal.sdk[row.harness] : null @@ -496,12 +499,65 @@ export function createProjectHarness(rt, { roleGithub } = {}) { await rt.save() return { ok: true } } + async function exitReceipt(row, action) { + const lease = ownEntry(rt.state.projectResourceLeases ?? {}, row.leaseId) + const resource = rt.projectResources.requireBinding(row.projectId, lease) + if (lease.projectId !== row.projectId || lease.attemptId !== row.attemptId || lease.resourceId !== row.resourceId || lease.role !== row.role || lease.generation !== row.generation || lease.worker?.handle !== row.id) throw refuse('SDK exit lease binding changed') + const identity = { version: SDK_EXIT_VERSION, root: resource.root, workerId: row.id, leaseId: row.leaseId, generation: row.generation } + const helper = join(resource.runtimeDir ?? resolve(import.meta.dirname, '../..'), 'src/host/project-sdk-exit.mjs') + const result = await rt.projectResources.command(row.projectId, row.resourceId, { cwd: resource.root, argv: ['node', helper, action, JSON.stringify(identity)], timeout: 10000 }) + return { lease, identity, result: JSON.parse(result.stdout) } + } + async function durableHostStop(row) { + if (row.exit?.code !== 0 || row.exit.signal || !row.eventLog || !row.lastEventHash) return false + try { + const root = await realpath(join(rt.home, 'project-artifacts', row.attemptId)), path = await realpath(row.eventLog), info = await lstat(row.eventLog) + if (!path.startsWith(root + '/') || path !== row.eventLog || !info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024) return false + const event = JSON.parse((await readFile(path, 'utf8')).trimEnd().split('\n').at(-1)) + return event.type === 'stopped' && event.released === true && sha(JSON.stringify(event)) === row.lastEventHash + } catch { return false } + } + async function reconcileExit(id, { stop = false } = {}) { + const row = ownEntry(workers(), id) + if (!row || active.has(id) && !active.get(id).closed) return false + const savedLease = ownEntry(rt.state.projectResourceLeases ?? {}, row.leaseId) + if (row.status === 'released' && savedLease?.status === 'released' && (confirmedWorkerExit(rt.state, row) || row.releaseProof?.kind === 'sdk-worker-spawn-failed' && row.releaseProof.workerId === id && row.releaseProof.generation === row.generation && savedLease.releaseProof?.kind === row.releaseProof.kind)) return true + let observed, source = 'resource-stopped-receipt' + try { + const persisted = row.status === 'released' && row.releaseProof?.kind === 'sdk-stop-ack-and-process-exit' && row.releaseProof.workerId === id && row.releaseProof.generation === row.generation + if (persisted || await durableHostStop(row)) { + const lease = ownEntry(rt.state.projectResourceLeases ?? {}, row.leaseId) + rt.projectResources.requireBinding(row.projectId, lease) + if (lease.projectId !== row.projectId || lease.attemptId !== row.attemptId || lease.resourceId !== row.resourceId || lease.role !== row.role || lease.worker?.handle !== id) return false + const identity = { version: SDK_EXIT_VERSION, workerId: id, leaseId: row.leaseId, generation: row.generation } + observed = { lease, identity, result: { ...identity, status: 'exited' } }; source = persisted ? 'persisted-sdk-exit-proof' : 'durable-host-stop-event' + } else if (row.exitRecovery?.version === SDK_EXIT_VERSION) observed = await exitReceipt(row, stop ? 'stop' : 'inspect') + else return false + } catch { return false } + const { lease, identity, result } = observed + if (row !== ownEntry(workers(), id) || lease !== ownEntry(rt.state.projectResourceLeases ?? {}, identity.leaseId) || row.leaseId !== identity.leaseId || row.generation !== identity.generation || lease.generation !== identity.generation || lease.worker?.handle !== id || active.has(id) && !active.get(id).closed || Object.entries(identity).some(([key, value]) => key !== 'root' && result[key] !== value) || result.status !== 'exited') return false + rt.projectResources.requireBinding(row.projectId, lease) + const proof = { kind: 'sdk-stop-ack-and-process-exit', source, workerId: id, generation: row.generation, at: Date.now() } + row.status = 'released'; row.releasedAt = proof.at; row.releaseProof = proof + if (row.githubAccess) { row.githubAccess.renewing = false; row.githubAccess.status = 'closed' } + if (row.scratchDirectory) row.scratchCleanupPending = true + clearTimeout(active.get(id)?.timer); active.delete(id) + lease.releaseProof = proof + await rt.projectResources.release(lease.id, lease.generation, { handle: id, exited: true, proof }) + await rt.projectDelivery.releaseSession(id) + await rt.save() + try { await exitReceipt(row, 'forget') } catch { /* Durable exit proof is retained even if artifact cleanup fails. */ } + return true + } + async function reconcileExits() { + for (const row of Object.values(workers())) await reconcileExit(row.id) + } async function release(id) { const row = ownEntry(workers(), id) if (!row) return false - if (row.status === 'released') return true + if (row.status === 'released') { await reconcileExit(id); return true } const control = active.get(id) - if (!control) throw refuse('SDK worker exit cannot be confirmed after restart') + if (!control || control.closed && !control.stoppedConfirmed && !control.spawnFailed) { if (await reconcileExit(id, { stop: true })) return true; throw refuse('SDK worker exit cannot be confirmed after restart; inspect its resource-side stop and process evidence') } row.status = 'stopping'; if (row.githubAccess) { row.githubAccess.renewing = false; row.githubAccess.status = 'stopping' }; clearTimeout(control.timer) if (!control.closed) control.process.stdin.end(JSON.stringify({ type: 'stop' }) + '\n') let timer @@ -521,6 +577,7 @@ export function createProjectHarness(rt, { roleGithub } = {}) { } catch { row.scratchCleanupPending = true } // Preserve cleanup debt without inventing an active worker after confirmed exit. } await rt.save() + if (row.exitRecovery?.supported) { try { await exitReceipt(row, 'forget') } catch { /* Retain cleanup debt without losing confirmed exit. */ } } return true } async function chiefTurn({ agent, delegation, definition, context, text, onResult }) { @@ -606,7 +663,7 @@ export function createProjectHarness(rt, { roleGithub } = {}) { return nativeGithub.get(id)?.runtime.env ?? {} } return { enableNativeGithubEnv() { nativeEnvReady = true }, roleGithubActive: id => nativeGithub.has(id), bindNativeGithub, closeNativeGithub, nativeGithubEnv, redactGithub: github.redact, profile, spawn, dispatch, correctInput, checkNativeInput, taskInputWake, release, input, chiefTurn, onResult: fn => { listeners.add(fn); return () => listeners.delete(fn) }, - capabilities: harnessCapabilities, owns: id => Boolean(ownEntry(workers(), id)), hasController: id => active.has(id) && !active.get(id).closed, versions: VERSIONS } + capabilities: harnessCapabilities, owns: id => Boolean(ownEntry(workers(), id)), hasController: id => active.has(id) && !active.get(id).closed, reconcileExit, reconcileExits, versions: VERSIONS } } export function install(rt) { rt.projectHarness = createProjectHarness(rt) diff --git a/patches/ds-bot/files/src/host/project-resources.js b/patches/ds-bot/files/src/host/project-resources.js index 823b45063..b3c539e8f 100644 --- a/patches/ds-bot/files/src/host/project-resources.js +++ b/patches/ds-bot/files/src/host/project-resources.js @@ -275,6 +275,7 @@ export function createProjectResources(rt, { priorExit = priorScopeExited } = {} return changed } async function reconcile() { + await rt.projectHarness?.reconcileExits?.() let changed = false for (const lease of Object.values(leases()).filter(active)) { if (lease.processScope?.hostPid === process.pid && (lease.worker?.kind === 'native' && rt.projectDelivery?.nativeSessions?.currentHost(lease.worker.botId) || lease.worker?.kind === 'sdk' && rt.projectHarness?.hasController(lease.worker.handle) || lease.worker?.kind === 'native-turn' && rt.projectAgents?.hasExchange(lease.worker.handle))) continue diff --git a/patches/ds-bot/files/src/host/project-sdk-exit.mjs b/patches/ds-bot/files/src/host/project-sdk-exit.mjs new file mode 100644 index 000000000..fa0134a6c --- /dev/null +++ b/patches/ds-bot/files/src/host/project-sdk-exit.mjs @@ -0,0 +1,79 @@ +// Resource-side stop acknowledgement plus exact process birth/exit, not Task success. +import { mkdir, lstat, realpath, readFile, writeFile, rename, unlink } from 'node:fs/promises' +import { join, isAbsolute } from 'node:path' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { pathToFileURL } from 'node:url' +import { captureProcessScope, priorHostProcessExited } from './project-process-proof.js' +const exec = promisify(execFile) +export const SDK_EXIT_VERSION = 'project-sdk-exit/v1' +function identity(value) { + if (value?.version !== SDK_EXIT_VERSION || !isAbsolute(value.root ?? '') || !/^[A-Za-z0-9_-]{1,120}$/.test(value.workerId ?? '') || !/^[A-Za-z0-9_-]{1,120}$/.test(value.leaseId ?? '') || !Number.isSafeInteger(value.generation) || value.generation < 1) throw Error('Invalid SDK exit identity') + return { version: SDK_EXIT_VERSION, workerId: value.workerId, leaseId: value.leaseId, generation: value.generation } +} +async function target(value, create = false) { + const expected = identity(value), root = await realpath(value.root), directory = join(root, '.flybot-sdk-exit') + if (create) await mkdir(directory, { mode: 0o700 }).catch(error => { if (error.code !== 'EEXIST') throw error }) + const info = await lstat(directory) + if (!info.isDirectory() || info.isSymbolicLink() || await realpath(directory) !== directory) throw Error('SDK exit directory changed') + return { expected, path: join(directory, expected.workerId + '.json') } +} +async function record(value) { + const { expected, path } = await target(value), info = await lstat(path) + if (!info.isFile() || info.isSymbolicLink() || info.size > 8192) throw Error('SDK exit record changed') + const row = JSON.parse(await readFile(path, 'utf8')) + if (Object.entries(expected).some(([key, field]) => row[key] !== field) || !['running', 'stopped'].includes(row.status)) throw Error('SDK exit record identity changed') + return { row, path } +} +export async function registerSdkExit(value) { + const { expected, path } = await target(value, true) + await writeFile(path, JSON.stringify({ ...expected, scope: await captureProcessScope(), status: 'running' }), { flag: 'wx', mode: 0o600 }) +} +export async function acknowledgeSdkStop(value) { + const { row, path } = await record(value) + if (row.scope.hostPid !== process.pid) throw Error('SDK exit writer changed') + const pending = path + '.stopped' + await writeFile(pending, JSON.stringify({ ...row, status: 'stopped' }), { flag: 'wx', mode: 0o600 }) + await rename(pending, path) +} +// pidfd pins the verified process; no signal fallback to a potentially reused PID. +const stopProcess = `import os,sys,json,signal +s=json.loads(sys.argv[1]) +fd=os.pidfd_open(s['hostPid']) +try: + if open('/proc/sys/kernel/random/boot_id').read().strip()!=s['bootId'] or os.readlink('/proc/self/ns/pid')!=s['pidNamespace']: sys.exit(2) + stat=open('/proc/'+str(s['hostPid'])+'/stat').read(); fields=stat[stat.rfind(')')+2:].split() + if fields[19]!=s['startTicks']: sys.exit(2) + signal.pidfd_send_signal(fd,signal.SIGTERM) +finally: os.close(fd) +` +export async function inspectSdkExit(value, { stop = false } = {}) { + let current + try { current = await record(value) } catch { return { status: 'unknown', reason: 'missing-or-invalid-receipt' } } + if (current.row.status === 'stopped' && await priorHostProcessExited(current.row.scope)) return { status: 'exited', ...identity(value), scope: current.row.scope } + if (!stop) return { status: 'unknown', reason: 'stop-or-process-exit-unconfirmed' } + try { + const scope = current.row.scope + if (!/^[a-f0-9-]{36}$/.test(scope?.bootId ?? '') || !/^\d+$/.test(scope?.startTicks ?? '') || !/^pid:\[\d+\]$/.test(scope?.pidNamespace ?? '') || !Number.isSafeInteger(scope.hostPid) || scope.hostPid < 1) return { status: 'unknown', reason: 'process-birth-unavailable' } + await exec('python3', ['-c', stopProcess, JSON.stringify(scope)], { timeout: 2000, maxBuffer: 8192, env: { PATH: process.env.PATH, LANG: 'C.UTF-8' } }) + } catch { return { status: 'unknown', reason: 'verified-stop-unavailable' } } + const deadline = Date.now() + 6000 + while (Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 100)) + const result = await inspectSdkExit(value) + if (result.status === 'exited') return result + } + return { status: 'unknown', reason: 'stop-or-process-exit-unconfirmed' } +} +export async function forgetSdkExit(value) { + const { path } = await record(value) + await unlink(path) +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const [action, text] = process.argv.slice(2), value = JSON.parse(text) + if (action === 'forget') { await forgetSdkExit(value); process.stdout.write('{}') } + else if (['inspect', 'stop'].includes(action)) process.stdout.write(JSON.stringify(await inspectSdkExit(value, { stop: action === 'stop' }))) + else throw Error('Unsupported SDK exit operation') + } catch { process.stdout.write(JSON.stringify({ status: 'unknown', reason: 'exit-evidence-unavailable' })); process.exitCode = 1 } +} diff --git a/patches/ds-bot/files/test/project-harness.test.js b/patches/ds-bot/files/test/project-harness.test.js index ffbf2bceb..70778a6b0 100644 --- a/patches/ds-bot/files/test/project-harness.test.js +++ b/patches/ds-bot/files/test/project-harness.test.js @@ -3,7 +3,10 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { mkdtemp, mkdir, writeFile, readFile, rm, stat, rename, symlink } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { join, resolve } from 'node:path' +import { createInterface } from 'node:readline' +import { once } from 'node:events' +import { createHash } from 'node:crypto' import { execFileSync } from 'node:child_process' import { createProjectScheduler } from '../src/host/project-scheduler.js' import { createProjectEffects } from '../src/host/project-effects.js' @@ -70,6 +73,77 @@ test('SDK workers reject unregistered Workspaces and missing explicit profiles', await assert.rejects(harness.spawn({ executionProjectId: 'p', executionWorkDir: cwd }), /explicitly authorized/) }) +for (const priorState of ['unreachable', 'released-worker', 'released-lease', 'stale-generation', 'legacy-no-proof']) test('controller-lost SDK release reconciles ' + priorState + ' using exact exit evidence', async t => { + const { rt, cwd } = await fixture(t) + const scope = JSON.parse(execFileSync(process.execPath, ['--input-type=module', '-e', `import {captureProcessScope} from ${JSON.stringify(new URL('../src/host/project-process-proof.js', import.meta.url).href)};console.log(JSON.stringify(await captureProcessScope()))`], { encoding: 'utf8', env: { PATH: process.env.PATH } })) + const id = 'sdk-controller-lost', root = rt.config.projectResources.r.root + for (const name of ['project-sdk-exit.mjs', 'project-process-proof.js']) await writeFile(join(rt.config.projectResources.r.runtimeDir, 'src/host', name), await readFile(new URL('../src/host/' + name, import.meta.url))) + await mkdir(join(root, '.flybot-sdk-exit')) + await writeFile(join(root, '.flybot-sdk-exit', id + '.json'), JSON.stringify({ version: 'project-sdk-exit/v1', workerId: id, leaseId: 'l', generation: 1, scope, status: 'stopped' })) + rt.state.projectHarnessWorkers = { [id]: { id, projectId: 'p', attemptId: 'a', role: 'developer', resourceId: 'r', leaseId: 'l', generation: 1, status: priorState === 'released-worker' ? 'released' : 'unreachable', cwd, exitRecovery: { version: 'project-sdk-exit/v1' } } } + if (priorState === 'released-worker') rt.state.projectHarnessWorkers[id].releaseProof = { kind: 'sdk-stop-ack-and-process-exit', workerId: id, generation: 1, at: Date.now() } + Object.assign(rt.state.projectResourceLeases.l, { status: priorState === 'released-lease' ? 'released' : 'unreachable', worker: { kind: 'sdk', handle: id } }) + const restored = createProjectHarness(rt) + if (priorState === 'legacy-no-proof') { + delete rt.state.projectHarnessWorkers[id].exitRecovery + await rm(join(root, '.flybot-sdk-exit', id + '.json')) + await assert.rejects(restored.release(id), /exit cannot be confirmed/) + assert.equal(rt.state.projectResourceLeases.l.status, 'unreachable') + delete rt.state.projectHarnessWorkers[id] + return + } + if (priorState === 'stale-generation') { + const command = rt.projectResources.command + rt.projectResources.command = async (...args) => { const result = await command(...args); rt.state.projectResourceLeases.l.generation = 2; return result } + await assert.rejects(restored.release(id), /exit cannot be confirmed/) + assert.equal(rt.state.projectResourceLeases.l.status, 'unreachable') + delete rt.state.projectHarnessWorkers[id] // No live process exists in this synthetic stale-binding fixture. + return + } + assert.equal(await restored.release(id), true) + assert.equal(rt.state.projectResourceLeases.l.status, 'released') + assert.equal(rt.state.projectHarnessWorkers[id].releaseProof.kind, 'sdk-stop-ack-and-process-exit') +}) + +for (const recovery of ['release', 'reconcile']) test('controller-lost host ' + recovery + ' waits for the actual resource worker to settle and exit', { timeout: 15000 }, async t => { + const { rt, cwd } = await fixture(t), id = 'sdk-live-controller-lost' + rt.config.projectResources.r.runtimeDir = resolve(import.meta.dirname, '..') + rt.state.projectHarnessWorkers = { [id]: { id, projectId: 'p', attemptId: 'a', role: 'developer', resourceId: 'r', leaseId: 'l', generation: 1, status: 'unreachable', cwd, exitRecovery: { version: 'project-sdk-exit/v1' } } } + await rt.projectResources.markStarted('l', 1, { kind: 'sdk', handle: id }) + const identity = { version: 'project-sdk-exit/v1', root: rt.config.projectResources.r.root, workerId: id, leaseId: 'l', generation: 1 } + const child = rt.projectResources.startProcess('p', 'r', { cwd, argv: ['node', resolve(import.meta.dirname, '../src/host/project-harness-worker.mjs'), JSON.stringify(identity)] }) + child.stderr.resume(); child.stdin.on('error', () => {}) + const exit = once(child, 'close'), lines = createInterface({ input: child.stdout }) + t.after(async () => { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); await exit; lines.close() }) + await once(lines, 'line') + rt.projectHarness = createProjectHarness(rt) + assert.equal(rt.projectResources.occupied('r'), 1) + if (recovery === 'release') assert.equal(await rt.projectHarness.release(id), true) + else { child.stdin.end(); await exit; await rt.projectResources.reconcile() } + await exit + assert.equal(rt.projectResources.occupied('r'), 0) + assert.equal(rt.state.projectHarnessWorkers[id].releaseProof.source, 'resource-stopped-receipt') + assert.equal(rt.state.tasks.t.status, undefined, 'Physical exit does not complete the Task') +}) + +for (const previousStatus of ['unreachable', 'released']) test('legacy ' + previousStatus + ' SDK reconciles only its saved stop event and zero process exit', async t => { + const { rt, harness, cwd } = await fixture(t) + const start = rt.projectResources.startProcess; let child + rt.projectResources.startProcess = (...args) => (child = start(...args)) + const id = await harness.spawn({ executionProjectId: 'p', executionWorkDir: cwd, prompt: 'Own legacy worker' }) + const exited = once(child, 'close') + child.stdin.end(JSON.stringify({ type: 'stop' }) + '\n'); await exited + const row = rt.state.projectHarnessWorkers[id] + const stoppedHash = createHash('sha256').update(JSON.stringify({ type: 'stopped', released: true })).digest('hex') + for (let n = 0; n < 100 && row.lastEventHash !== stoppedHash; ++n) await new Promise(resolve => setTimeout(resolve, 5)) + assert.equal(row.lastEventHash, stoppedHash) + delete row.exitRecovery; row.status = previousStatus; rt.state.projectResourceLeases.l.status = previousStatus + rt.projectHarness = createProjectHarness(rt) + await rt.projectResources.reconcile() + assert.equal(row.status, 'released'); assert.equal(rt.projectResources.occupied('r'), 0) + assert.equal(row.releaseProof.source, 'durable-host-stop-event') +}) + test('non-App Dev receives Git mutation rights only for its bound isolated resource clone', async t => { const { harness, rt, cwd } = await fixture(t) execFileSync('git', ['init', '-q', cwd]) diff --git a/patches/ds-bot/files/test/project-sdk-exit-sdk.test.js b/patches/ds-bot/files/test/project-sdk-exit-sdk.test.js new file mode 100644 index 000000000..7bfcc2c4d --- /dev/null +++ b/patches/ds-bot/files/test/project-sdk-exit-sdk.test.js @@ -0,0 +1,44 @@ +// Opt-in actual SDK/CLI abort and worker exit; provider is a held local fixture. +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { createServer } from 'node:http' +import { spawn, execFileSync } from 'node:child_process' +import { once } from 'node:events' +import { createInterface } from 'node:readline' +import { mkdtemp, mkdir, rm, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { inspectSdkExit, SDK_EXIT_VERSION } from '../src/host/project-sdk-exit.mjs' +if (process.env.FLYBOT_TEST_EXIT_SDK === '1') test('actual Codex SDK settles after controller pipe loss and leaves verifiable resource exit', { timeout: 45000 }, async () => { + const root = await mkdtemp(join(tmpdir(), 'flybot-sdk-exit-actual-')), cwd = join(root, 'work'), home = join(root, 'home') + await mkdir(cwd); await mkdir(home) + execFileSync('git', ['init', '-q', cwd]) + let reached + const called = new Promise(resolve => { reached = resolve }) + const server = createServer((request, response) => { request.resume(); reached(); response.writeHead(200, { 'content-type': 'text/event-stream' }); response.write(': hold until SDK abort\n\n') }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const identity = { version: SDK_EXIT_VERSION, root, workerId: 'actual-sdk-worker', leaseId: 'actual-sdk-lease', generation: 1 } + const child = spawn(process.execPath, [resolve(import.meta.dirname, '../src/host/project-harness-worker.mjs'), JSON.stringify(identity)], { env: { PATH: process.env.PATH }, stdio: ['pipe', 'pipe', 'pipe'] }) + const exit = once(child, 'close'), lines = createInterface({ input: child.stdout }) + child.stderr.resume(); child.stdin.on('error', () => {}) + try { + const [first] = await once(lines, 'line'), ready = JSON.parse(first) + assert.equal(ready.sdk.codex, '0.166.0') + child.stdin.write(JSON.stringify({ type: 'start', settings: { harness: 'codex', cwd, model: 'fixture-model', noTools: true, env: { HOME: home, CODEX_HOME: home, PATH: process.env.PATH }, config: { model_provider: 'fixture', model_providers: { fixture: { name: 'fixture', base_url: `http://127.0.0.1:${server.address().port}`, wire_api: 'responses', requires_openai_auth: false } } } }, prompt: 'Wait for this isolated fixture response' }) + '\n') + await Promise.race([called, new Promise((_, reject) => setTimeout(() => reject(Error('Actual SDK did not reach fixture provider')), 20000).unref())]) + const cliPids = (await readFile(`/proc/${child.pid}/task/${child.pid}/children`, 'utf8')).trim().split(/\s+/).filter(Boolean) + assert.ok(cliPids.length, 'Actual SDK spawned its CLI') + assert.equal((await inspectSdkExit(identity)).status, 'unknown') + child.stdout.destroy(); child.stdin.end() + const [code] = await exit + assert.equal(code, 0) + assert.equal((await inspectSdkExit(identity)).status, 'exited', 'SDK abort must settle before the actual worker acknowledges stop') + for (const pid of cliPids) { + try { const stat = await readFile(`/proc/${pid}/stat`, 'utf8'); assert.ok(['Z', 'X'].includes(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0]), 'SDK CLI must also have exited') } + catch (error) { if (error.code !== 'ENOENT') throw error } + } + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + await exit; lines.close(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); await rm(root, { recursive: true, force: true }) + } +}) diff --git a/patches/ds-bot/files/test/project-sdk-exit.test.js b/patches/ds-bot/files/test/project-sdk-exit.test.js new file mode 100644 index 000000000..add8de59a --- /dev/null +++ b/patches/ds-bot/files/test/project-sdk-exit.test.js @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { spawn, execFileSync } from 'node:child_process' +import { createInterface } from 'node:readline' +import { once } from 'node:events' +import { mkdtemp, mkdir, rm, readFile, writeFile, symlink, unlink } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { SDK_EXIT_VERSION, registerSdkExit, acknowledgeSdkStop, inspectSdkExit, forgetSdkExit } from '../src/host/project-sdk-exit.mjs' +async function fixture(t) { + const root = await mkdtemp(join(tmpdir(), 'flybot-sdk-exit-')) + t.after(() => rm(root, { recursive: true, force: true })) + const value = { version: SDK_EXIT_VERSION, root, workerId: 'sdk-exit-fixture', leaseId: 'lease-fixture', generation: 1 } + return { value, path: join(root, '.flybot-sdk-exit', value.workerId + '.json') } +} +async function worker(t) { + const f = await fixture(t) + const child = spawn(process.execPath, [fileURLToPath(new URL('../src/host/project-harness-worker.mjs', import.meta.url)), JSON.stringify(f.value)], { env: { PATH: process.env.PATH }, stdio: ['pipe', 'pipe', 'pipe'] }) + child.stderr.resume(); child.stdin.on('error', () => {}) + const exit = once(child, 'close'), lines = createInterface({ input: child.stdout }) + t.after(async () => { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); await exit; lines.close() }) + const [line] = await Promise.race([once(lines, 'line'), exit.then(() => { throw Error('Worker did not register') })]) + assert.equal(JSON.parse(line).exitRecovery, SDK_EXIT_VERSION) + return { ...f, child, exit } +} +test('invalid/missing SDK exit evidence never implies release and registration cannot overwrite another record', async t => { + const f = await fixture(t) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + await assert.rejects(registerSdkExit({ ...f.value, workerId: '../foreign' }), /identity/) + await registerSdkExit(f.value) + await assert.rejects(registerSdkExit(f.value), /EEXIST/) + assert.equal((await inspectSdkExit({ ...f.value, generation: 2 }, { stop: true })).status, 'unknown') + await acknowledgeSdkStop(f.value) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown', 'A stopped receipt alone cannot prove the writer process exited') + await writeFile(f.path, 'malformed receipt') + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + await rm(f.path); await symlink(join(f.value.root, 'private'), f.path) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + await unlink(f.path); await writeFile(f.path, 'x'.repeat(9000)) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') +}) +test('SDK exit records refuse a symlinked supervisor directory', async t => { + const f = await fixture(t), foreign = join(f.value.root, 'foreign') + await mkdir(foreign); await symlink(foreign, join(f.value.root, '.flybot-sdk-exit')) + await assert.rejects(registerSdkExit(f.value), /directory changed/) +}) +test('resource exit CLI rejects unsupported operations with a bounded machine-readable result', () => { + assert.throws(() => execFileSync(process.execPath, [fileURLToPath(new URL('../src/host/project-sdk-exit.mjs', import.meta.url)), 'unsupported', '{}'], { encoding: 'utf8', stdio: 'pipe' }), error => { + assert.deepEqual(JSON.parse(error.stdout), { status: 'unknown', reason: 'exit-evidence-unavailable' }); return true + }) +}) +if (process.platform === 'linux') { + test('actual idle SDK worker can be stopped by exact resource process identity after controller loss', { timeout: 15000 }, async t => { + const f = await worker(t) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + assert.equal((await inspectSdkExit(f.value, { stop: true })).status, 'exited') + await f.exit + assert.equal(JSON.parse(await readFile(f.path, 'utf8')).status, 'stopped') + await forgetSdkExit(f.value) + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + }) + test('closed controller pipes still produce a durable settled stop receipt', { timeout: 10000 }, async t => { + const f = await worker(t) + f.child.stdout.destroy(); f.child.stdin.end() + await f.exit + assert.equal((await inspectSdkExit(f.value)).status, 'exited') + }) + test('a killed worker without settled stop acknowledgement remains unknown', { timeout: 10000 }, async t => { + const f = await worker(t) + f.child.kill('SIGKILL'); await f.exit + assert.equal((await inspectSdkExit(f.value)).status, 'unknown') + }) + test('a process that cannot acknowledge settled shutdown retains unknown exit after the stop budget', { timeout: 10000 }, async t => { + const f = await fixture(t) + const module = new URL('../src/host/project-sdk-exit.mjs', import.meta.url).href + const child = spawn(process.execPath, ['--input-type=module', '-e', `import {registerSdkExit} from ${JSON.stringify(module)};process.on('SIGTERM',()=>{});await registerSdkExit(${JSON.stringify(f.value)});console.log('ready');setInterval(()=>{},1000)`], { env: { PATH: process.env.PATH }, stdio: ['pipe', 'pipe', 'pipe'] }) + const exit = once(child, 'close'), lines = createInterface({ input: child.stdout }); child.stderr.resume() + t.after(async () => { child.kill('SIGKILL'); await exit; lines.close() }) + await once(lines, 'line') + assert.equal((await inspectSdkExit(f.value, { stop: true })).status, 'unknown') + assert.equal(child.exitCode, null) + assert.equal(JSON.parse(await readFile(f.path, 'utf8')).status, 'running') + }) + test('a mismatched birth cannot signal a reused PID or release another generation', { timeout: 15000 }, async t => { + const f = await worker(t), original = await readFile(f.path, 'utf8'), row = JSON.parse(original) + row.scope.startTicks = String(Number(row.scope.startTicks) + 1) + await writeFile(f.path, JSON.stringify(row)) + assert.equal((await inspectSdkExit(f.value, { stop: true })).status, 'unknown') + assert.equal(f.child.exitCode, null) + await writeFile(f.path, original) + assert.equal((await inspectSdkExit(f.value, { stop: true })).status, 'exited') + await f.exit + }) +}