From 67c674fa11f47c98e6b172620830439ee85bee9c Mon Sep 17 00:00:00 2001 From: James3014 <185240413+James3014@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:34:26 +0800 Subject: [PATCH 1/4] refactor: retire active Carrier governance for direct operations (C4 #471) --- docs/coordination-readers.md | 15 ++- src/cli.ts | 2 + src/cutover-state.test.ts | 29 +++++ src/cutover-state.ts | 31 ++++- src/durable-operations.test.ts | 201 +++++++++++++++++++++++++++++++++ src/durable-operations.ts | 138 +++++++++++++++++++--- src/mcp-cutover.ts | 4 +- src/server.test.ts | 128 +++++++++++++++++++-- src/server.ts | 53 ++++++--- 9 files changed, 557 insertions(+), 44 deletions(-) diff --git a/docs/coordination-readers.md b/docs/coordination-readers.md index a5f2977ed..a241cc187 100644 --- a/docs/coordination-readers.md +++ b/docs/coordination-readers.md @@ -1,6 +1,8 @@ -# CLI coordination readers +# Legacy CLI coordination readers (migration compatibility) -An operator can connect the existing host reader interface when starting DevSpace: +**C4 migration boundary:** New `cutover_start` and single-conversation or managed-worktree `dependency_sync` use DevSpace-native durable operations, not Carrier/ControlPlane admission. Ordinary startup does not initialize the Carrier store unless a historical persisted bound effect still needs recovery, or an operator explicitly supplies legacy completion/coordination readers. The commands below document historical compatibility, not a supported path for introducing new governance. Do not use them to start new cutovers or direct dependency operations. + +An operator recovering a historical bound effect can connect its original trusted host reader interface when starting DevSpace: ```sh devspace serve --coordination-reader-module /absolute/path/readers.mjs --coordination-reader-sha256 <64-hex-sha256> @@ -49,10 +51,11 @@ Pairing remains available. Selecting the module pins executable code, not eviden truth: the reviewed reader must still validate its canonical sources, artifact integrity and acceptance provenance. Synthetic fixtures cannot establish delivery. -Without a custom reader module, the server now exposes a local pairing path for -bounded dependency operations. Unpaired sessions have no coordination authority. -An explicitly configured custom reader retains exclusive control; the built-in -pairing tools are not enabled alongside it. +The following pairing workflow describes **historical persisted Carrier sessions**. +It is no longer the ordinary MCP execution path, and `coordination_pair` / +`coordination_resume` are not registered as new direct-operation tools. +A custom legacy reader remains solely responsible for the original authority +of a pre-existing bound record; it cannot authorize a new direct operation. 1. The MCP client calls `coordination_pair` and keeps its private credential. The pending ID identifies the pairing request; it does not grant authority. diff --git a/src/cli.ts b/src/cli.ts index 845e8aadb..21cf5cc19 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -437,6 +437,8 @@ function printHelp(): void { " devspace config set publicBaseUrl ", " devspace models List available models in catalog", " devspace models refresh Refresh current provider model catalog and bump generation", + "", + "Legacy Carrier administration (persisted bound recovery only; not needed for new direct execution):", " devspace carrier inspect ", " devspace carrier show ", " devspace carrier reauthorize --validity-version --until ", diff --git a/src/cutover-state.test.ts b/src/cutover-state.test.ts index a7284ce41..b3309ac14 100644 --- a/src/cutover-state.test.ts +++ b/src/cutover-state.test.ts @@ -1182,6 +1182,35 @@ for (const missing of ["witnessCutoverId", "witnessServerInstanceId", "witnessEx const coordinationBinding = {leaseId:"lease-one",pinnedLeaseVersion:2,operationHandle:"operation-one",requestHash:"a".repeat(64),ownerThread:"controller-one"}; +test("C4 direct correlation cannot be malformed, mixed with Carrier, or changed during drain", () => { + const root = mkdtempSync(join(tmpdir(), "devspace-c4-direct-correlation-")); + try { + const store = new CutoverStateStore(root); + const directOperation = { operationId: "op_" + "a".repeat(16), requestHash: "b".repeat(64) }; + const base = { oldServerIdentity: oldIdentity, expectedNewIdentity: { sourceCommit: "target", buildId: "target" } }; + assert.throws(() => store.begin({ ...base, coordinationBinding, directOperation }), /cannot mix/i); + assert.throws(() => store.begin({ ...base, directOperation: { ...directOperation, requestHash: "bad" } }), /malformed direct/i); + assert.deepEqual(readdirSync(root), []); + const created = store.begin({ ...base, directOperation }); + assert.equal(created.coordinationBinding, undefined); + assert.deepEqual(new CutoverStateStore(root).get()?.directOperation, directOperation); + store.recordDrain(created.cutoverId, { activeSessions: 0, oldestAgeMs: 0 }); + const active = join(root, "cutover", "active"); + const drain = join(active, readdirSync(active).find(name => name.startsWith("drained-"))!); + const original = readFileSync(drain, "utf8"); + for (const tamper of [undefined, { ...directOperation, requestHash: "c".repeat(64) }]) { + const changed = JSON.parse(original); + if (tamper) changed.directOperation = tamper; else delete changed.directOperation; + writeFileSync(drain, JSON.stringify(changed)); + assert.throws(() => new CutoverStateStore(root).get(), /correlation changed/i); + writeFileSync(drain, original); + } + assert.deepEqual(new CutoverStateStore(root).get()?.directOperation, directOperation); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("coordination binding is validated before writes and survives restart and drain", () => { const root=mkdtempSync(join(tmpdir(),"devspace-cutover-correlation-")); try { diff --git a/src/cutover-state.ts b/src/cutover-state.ts index 74dde9ac0..90d78eacc 100644 --- a/src/cutover-state.ts +++ b/src/cutover-state.ts @@ -321,9 +321,17 @@ export interface CutoverCoordinationBinding { readonly ownerThread: string; } +/** Correlation only: an OWNER_DIRECT cutover does not grant carrier authority. */ +export interface CutoverDirectOperationBinding { + readonly operationId: string; + readonly requestHash: string; +} + export interface DurableCutoverRecord { /** Trusted consumer correlation only; never issues authority. */ coordinationBinding?: Readonly; + /** Exact durable intent of a new OWNER_DIRECT generation; not an authority grant. */ + directOperation?: Readonly; schema: typeof CUTOVER_STATE_SCHEMA; cutoverId: string; phase: "prepared" | "drained" | "closed" | "superseded"; @@ -484,8 +492,11 @@ export class CutoverStateStore { expectedNewIdentity: ExpectedCutoverIdentity; expiresAt?: string; coordinationBinding?: CutoverCoordinationBinding; + directOperation?: CutoverDirectOperationBinding; }): DurableCutoverRecord { const coordinationBinding = input.coordinationBinding === undefined ? undefined : validatedCoordinationBinding(input.coordinationBinding); + const directOperation = input.directOperation === undefined ? undefined : validatedDirectOperationBinding(input.directOperation); + if (coordinationBinding && directOperation) throw new CutoverStateError("Cutover cannot mix Carrier authority with OWNER_DIRECT intent."); mkdirSync(this.cutoverRoot, { recursive: true, mode: 0o700 }); const now = new Date(this.now()).toISOString(); const record: DurableCutoverRecord = { @@ -493,6 +504,7 @@ export class CutoverStateStore { cutoverId: this.newId(), phase: "prepared", ...(coordinationBinding ? { coordinationBinding } : {}), + ...(directOperation ? { directOperation } : {}), oldServerIdentity: input.oldServerIdentity, expectedNewIdentity: input.expectedNewIdentity, createdAt: now, @@ -1627,9 +1639,22 @@ function validatedCoordinationBinding(value: unknown): Readonly; } +function validatedDirectOperationBinding(value: unknown): Readonly { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new CutoverStateError("Malformed direct cutover operation binding."); + const binding = value as Record; + if (Object.keys(binding).length !== 2 || + !/^op_[a-f0-9]{16}$/.test(String(binding.operationId)) || + !/^[a-f0-9]{64}$/.test(String(binding.requestHash))) { + throw new CutoverStateError("Malformed direct cutover operation binding."); + } + return Object.freeze({operationId: binding.operationId as string, requestHash: binding.requestHash as string}); +} + function assertSameCoordinationBinding(created: DurableCutoverRecord, event: DurableCutoverRecord): void { - if (created.cutoverId !== event.cutoverId || !isDeepStrictEqual(created.coordinationBinding, event.coordinationBinding)) { - throw new CutoverStateError("Cutover coordination binding changed within its generation."); + if (created.cutoverId !== event.cutoverId || + !isDeepStrictEqual(created.coordinationBinding, event.coordinationBinding) || + !isDeepStrictEqual(created.directOperation, event.directOperation)) { + throw new CutoverStateError("Cutover coordination binding or direct operation correlation changed within its generation."); } } @@ -1663,6 +1688,8 @@ function parseRecord(raw: string): DurableCutoverRecord { } const record = value as DurableCutoverRecord; if (record.coordinationBinding !== undefined) record.coordinationBinding = validatedCoordinationBinding(record.coordinationBinding); + if (record.directOperation !== undefined) record.directOperation = validatedDirectOperationBinding(record.directOperation); + if (record.coordinationBinding && record.directOperation) throw new CutoverStateError("Cutover cannot mix Carrier authority with OWNER_DIRECT intent."); if (record.bindingRepair) { assertValidBindingRepair(record.bindingRepair, record); } diff --git a/src/durable-operations.test.ts b/src/durable-operations.test.ts index aa9187949..28cbb4f67 100644 --- a/src/durable-operations.test.ts +++ b/src/durable-operations.test.ts @@ -11,11 +11,14 @@ import { promisify } from "node:util"; import type { ControlPlaneOwnershipStore } from "./control-plane-ownership.js"; import type { ControlPlaneConsumerOptions } from "./control-plane-consumer.js"; import { loadConfig } from "./config.js"; +import { CutoverStateStore } from "./cutover-state.js"; import { canonicalizePath } from "./roots.js"; import { DurableOperationError, DurableOperationManager, DurableOperationStore, + planCutoverStart, + hashJson, type CommandRunner, } from "./durable-operations.js"; @@ -404,6 +407,167 @@ test("dependency_sync frozen recipe succeeds without changing manifest or lock i } }); +test("C4: new direct cutover persists exact durable correlation without Carrier governance", async () => { + const f = await fixture(); + const manager = new DurableOperationManager(f.config); + try { + const input = { + attemptKey: "c4-direct-cutover-1", + currentIdentity: { + serverInstanceId: "old-server", + sourceCommit: "a".repeat(40), + buildId: "old-build", + capabilityManifestSha256: "c".repeat(64), + }, + expectedIdentity: { + sourceCommit: "b".repeat(40), + buildId: "new-build", + capabilityManifestSha256: "c".repeat(64), + }, + }; + const first = manager.startDirectCutover(input); + assert.equal(first.status, "succeeded"); + assert.equal(first.retrySafe, false); + const cutover = new CutoverStateStore(f.config.stateDir).get(); + assert.equal(cutover?.phase, "prepared"); + assert.equal(cutover?.coordinationBinding, undefined); + assert.deepEqual(cutover?.directOperation, { + operationId: first.operationId, + requestHash: first.requestHash, + }); + assert.equal(first.receipt?.cutoverId, cutover?.cutoverId); + assert.deepEqual(manager.startDirectCutover(input), first); + assert.deepEqual(await manager.reconcile(first.operationId), first); + assert.throws(() => manager.startDirectCutover({ + ...input, + expectedIdentity: { ...input.expectedIdentity, buildId: "wrong-target" }, + }), /OPERATION_REPLAY_CONFLICT|materially different/); + assert.equal(new CutoverStateStore(f.config.stateDir).get()?.cutoverId, cutover?.cutoverId); + } finally { + manager.close(); + await f.cleanup(); + } +}); + +test("C4 refuses to downgrade a persisted Carrier-bound cutover into new OWNER_DIRECT authority", async () => { + const f = await fixture(); + const manager = new DurableOperationManager(f.config); + try { + const input = { + attemptKey: "c4-must-not-downgrade", + currentIdentity: { serverInstanceId: "legacy-old", sourceCommit: "a".repeat(40), buildId: "old" }, + expectedIdentity: { sourceCommit: "b".repeat(40), buildId: "new" }, + }; + const coordinationBinding = { + leaseId: "lease-old", + pinnedLeaseVersion: 2, + operationHandle: "op_" + "e".repeat(16), + requestHash: "f".repeat(64), + ownerThread: "legacy-controller", + }; + const bound = new CutoverStateStore(f.config.stateDir).begin({ + oldServerIdentity: input.currentIdentity, + expectedNewIdentity: input.expectedIdentity, + coordinationBinding, + }); + const attempt = manager.startDirectCutover(input); + assert.equal(attempt.status, "outcome_unknown"); + assert.equal(attempt.retrySafe, false); + assert.deepEqual(new CutoverStateStore(f.config.stateDir).get()?.coordinationBinding, coordinationBinding); + assert.equal(new CutoverStateStore(f.config.stateDir).get()?.directOperation, undefined); + assert.equal(new CutoverStateStore(f.config.stateDir).get()?.cutoverId, bound.cutoverId); + assert.throws(() => manager.reconcileDirectCutoverStart(attempt.operationId), /no retry is authorized/); + } finally { + manager.close(); + await f.cleanup(); + } +}); + +test("C4 direct cutover lost-ACK reconciles exact intent, never recreates absent or mismatched physical effects", async () => { + const f = await fixture(); + const manager = new DurableOperationManager(f.config); + try { + const input = { + attemptKey: "c4-unknown-cutover", + currentIdentity: { serverInstanceId: "old-1", sourceCommit: "a".repeat(40), buildId: "old-build" }, + expectedIdentity: { sourceCommit: "b".repeat(40), buildId: "new-build" }, + }; + const planned = planCutoverStart(f.config.stateDir, input); + const request = { ...planned.request, ownerDirect: true }; + const requestHash = hashJson(request); + const intent = manager.store.createOrReplay({ + operationId: planned.operationId, + attemptKey: input.attemptKey, + requestHash, + kind: "cutover_start", + scopeRoot: planned.stateRoot, + request, + }); + assert.equal(intent.record.status, "started"); + assert.throws(() => manager.reconcileDirectCutoverStart(planned.operationId), /no retry is authorized/); + assert.throws(() => manager.startDirectCutover(input), /no retry is authorized/); + assert.equal(new CutoverStateStore(f.config.stateDir).get(), undefined); + assert.equal(manager.store.getByOperationId(planned.operationId)?.status, "started"); + const observed = new CutoverStateStore(f.config.stateDir).begin({ + oldServerIdentity: input.currentIdentity, + expectedNewIdentity: input.expectedIdentity, + directOperation: { operationId: planned.operationId, requestHash }, + }); + const reconciled = manager.reconcileDirectCutoverStart(planned.operationId); + assert.equal(reconciled.status, "succeeded"); + assert.equal(reconciled.receipt?.cutoverId, observed.cutoverId); + assert.deepEqual(manager.startDirectCutover(input), reconciled); + } finally { + manager.close(); + await f.cleanup(); + } +}); + +test("C4 startup Carrier compatibility inventory excludes direct records but detects bound history", async () => { + const f = await fixture(); + const manager = new DurableOperationManager(f.config); + try { + assert.equal(manager.store.hasLegacyBoundOperations(), false); + manager.startDirectCutover({ + attemptKey: "c4-new-only", + currentIdentity: { serverInstanceId: "old", sourceCommit: "a".repeat(40), buildId: "old" }, + expectedIdentity: { sourceCommit: "b".repeat(40), buildId: "new" }, + }); + assert.equal(manager.store.hasLegacyBoundOperations(), false); + const boundSync = manager.store.createOrReplay({ + operationId: "op_1111111111111111", + attemptKey: "historical-bound-sync", + requestHash: "c".repeat(64), + kind: "dependency_sync", + scopeRoot: f.config.stateDir, + request: { version: "historical", workspaceRoot: f.config.stateDir }, + }); + assert.equal(manager.store.hasLegacyBoundOperations(), true); + manager.store.finish(boundSync.record.operationId, { status: "succeeded", retrySafe: false }); + assert.equal(manager.store.hasLegacyBoundOperations(), false, "terminal dependency sync must not keep Carrier resident"); + + const boundCutover = manager.store.createOrReplay({ + operationId: "op_2222222222222222", + attemptKey: "historical-bound-cutover", + requestHash: "d".repeat(64), + kind: "cutover_start", + scopeRoot: f.config.stateDir, + request: { version: "historical", coordinationBinding: { + leaseId: "lease-legacy", operationHandle: "op_2222222222222222", + requestHash: "d".repeat(64), ownerThread: "legacy-controller", pinnedLeaseVersion: 1, + } }, + }); + assert.equal(manager.store.hasLegacyBoundOperations(), true); + manager.store.finish(boundCutover.record.operationId, { + status: "succeeded", retrySafe: false, receipt: { lifecycleTerminal: true }, + }); + assert.equal(manager.store.hasLegacyBoundOperations(), false, "terminal cutover must not keep Carrier resident"); + } finally { + manager.close(); + await f.cleanup(); + } +}); + test("dependency_sync OWNER_DIRECT isolated mode does not require carrier authority", async () => { const f = await fixture(); try { @@ -445,6 +609,43 @@ test("dependency_sync OWNER_DIRECT isolated mode does not require carrier author } }); +test("C4 checkout reconciliation rejects a forged isolated witness after lost process acknowledgement", async () => { + const f = await fixture(); + try { + const project = join(f.root, "checkout"); + await mkdir(project); + await writeFile(join(project, "package.json"), JSON.stringify({ name: "fixture", version: "1.0.0" }) + "\n"); + await writeFile(join(project, "package-lock.json"), JSON.stringify({ name: "fixture", version: "1.0.0", lockfileVersion: 3, packages: {} }) + "\n"); + await git(project, "init"); + await git(project, "config", "user.email", "devspace@example.com"); + await git(project, "config", "user.name", "DevSpace Test"); + await git(project, "add", "."); + await git(project, "commit", "-m", "fixture"); + const manager = new DurableOperationManager(f.config, async () => { + throw new Error("lost checkout process acknowledgement"); + }); + try { + const unknown = await manager.dependencySync({ + attemptKey: "c4-checkout-no-witness", + workspaceId: "ws_single_checkout", + workspaceRoot: project, + recipe: "npm_ci", + ownerDirectExecution: true, + }); + assert.equal(unknown.status, "outcome_unknown"); + assert.equal(unknown.retrySafe, false); + assert.equal(unknown.request.ownerDirectExecution, true); + manager.store.recordDependencyTerminal(unknown.operationId, unknown.requestHash, "OWNER_DIRECT_ISOLATED", 0, true); + assert.throws(() => manager.reconcileOwnerDirectDependencySync(unknown.operationId), /No exact terminal witness exists/); + assert.equal(manager.store.getByOperationId(unknown.operationId)?.status, "outcome_unknown"); + } finally { + manager.close(); + } + } finally { + await f.cleanup(); + } +}); + test("dependency_sync OWNER_DIRECT isolated reconciliation requires its exact terminal witness", async () => { const f = await fixture(); try { diff --git a/src/durable-operations.ts b/src/durable-operations.ts index 624b98889..c4bfca55a 100644 --- a/src/durable-operations.ts +++ b/src/durable-operations.ts @@ -200,6 +200,21 @@ export class DurableOperationStore { return rows.map(rowToRecord); } + /** Lightweight startup query; historical records remain immutable. */ + hasLegacyBoundOperations(): boolean { + return this.database.sqlite.prepare(` + select 1 from durable_operations + where (kind = 'cutover_start' + and json_type(request_json, '$.coordinationBinding') = 'object' + and (status != 'succeeded' or json_extract(receipt_json, '$.lifecycleTerminal') is not 1)) + or (kind = 'dependency_sync' + and status in ('started', 'outcome_unknown') + and json_extract(request_json, '$.ownerDirectIsolated') is not 1 + and json_extract(request_json, '$.ownerDirectExecution') is not 1) + limit 1 + `).get() !== undefined; + } + getByAttempt(scopeRoot: string, attemptKey: string): DurableOperationRecord | undefined { const row = this.database.sqlite.prepare( "select * from durable_operations where scope_root = ? and attempt_key = ? limit 1", @@ -330,8 +345,11 @@ export interface DependencySyncInput { * Trusted server-side admission only. Callers cannot set this through MCP. * Allows OWNER_DIRECT frozen dependency sync without a carrier only for a * DevSpace-managed isolated worktree already admitted for this conversation. + * Kept as a persisted discriminator for older direct operations. */ ownerDirectIsolated?: boolean; + /** Trusted server-admitted, single-conversation checkout; never caller-supplied. */ + ownerDirectExecution?: boolean; } export type CommandRunner = ( @@ -591,6 +609,91 @@ export class DurableOperationManager { }); } + /** + * New cutovers are DevSpace-native effects, not Carrier-governed actions. + * The intent is durably recorded before the cutover file is written; the + * two independent stores are tied by an exact operation/request digest. + */ + startDirectCutover(input: CutoverStartInput): DurableOperationRecord { + const planned = planCutoverStart(this.config.stateDir, input); + const { snapshot, stateRoot, operationId } = planned; + const request = { ...planned.request, ownerDirect: true }; + const requestHash = hashJson(request); + const intent = this.store.createOrReplay({ + operationId, + attemptKey: snapshot.attemptKey, + requestHash, + kind: "cutover_start", + scopeRoot: stateRoot, + request, + }); + if (!intent.created) return this.reconcileDirectCutoverStart(operationId); + try { + return this.store.atomic(() => { + if (!isDeepStrictEqual(this.store.getByOperationId(operationId), intent.record)) { + throw new ControlPlaneOwnershipError("CAS_CONFLICT", "direct cutover intent changed before file write"); + } + const controller = new McpCutoverController(new CutoverStateStore(stateRoot), snapshot.currentIdentity); + controller.begin(snapshot.expectedIdentity, snapshot.expiresAt, undefined, { operationId, requestHash }); + return this.reconcileDirectCutoverStart(operationId); + }); + } catch (error) { + return this.store.atomic(() => { + if (!isDeepStrictEqual(this.store.getByOperationId(operationId), intent.record)) { + throw new ControlPlaneOwnershipError("CAS_CONFLICT", "late direct cutover result cannot replace newer durable state"); + } + return this.store.finish(operationId, { + status: "outcome_unknown", + retrySafe: false, + errorCode: "RECONCILIATION_REQUIRED", + errorMessage: error instanceof Error ? error.message : String(error), + }); + }); + } + } + + reconcileDirectCutoverStart(operationId: string): DurableOperationRecord { + return this.store.atomic(() => { + const record = this.store.getByOperationId(operationId); + if (!record || record.kind !== "cutover_start" || + record.scopeRoot !== canonicalizePath(this.config.stateDir) || + record.request.ownerDirect !== true || record.request.coordinationBinding !== undefined || + record.request.stateRoot !== record.scopeRoot || + typeof record.request.baseRevision !== "string" || + hashJson(record.request) !== record.requestHash) { + throw new ControlPlaneOwnershipError("CAS_CONFLICT", "direct cutover intent or resource identity is invalid"); + } + const expectedBinding = { operationId, requestHash: record.requestHash }; + const verifiedReceipt = record.receipt && + isDeepStrictEqual(record.receipt.directOperation, expectedBinding) && + record.receipt.startVerified === true && + typeof record.receipt.cutoverId === "string"; + // A terminally recorded start can be replayed after its generation has + // closed/archived. This returns only prior evidence, never a new effect. + if (record.status === "succeeded" && verifiedReceipt) return record; + const observed = new CutoverStateStore(record.scopeRoot).get(); + if (!observed || observed.coordinationBinding !== undefined || + !isDeepStrictEqual(observed.directOperation, expectedBinding) || + !isDeepStrictEqual(observed.oldServerIdentity, record.request.currentIdentity) || + !isDeepStrictEqual(observed.expectedNewIdentity, record.request.expectedIdentity) || + observed.expiresAt !== record.request.expiresAt) { + throw new DurableOperationError( + "RECONCILIATION_REQUIRED", + "direct cutover lacks exact persisted operation-to-file correlation; no retry is authorized", + record, + ); + } + if (!isDeepStrictEqual(this.store.getByOperationId(operationId), record)) { + throw new ControlPlaneOwnershipError("CAS_CONFLICT", "direct cutover intent changed during physical readback"); + } + return this.store.finish(operationId, { + status: "succeeded", + retrySafe: false, + receipt: { cutoverId: observed.cutoverId, directOperation: expectedBinding, startVerified: true, lifecycleTerminal: false }, + }); + }); + } + startCutover(input: CutoverStartInput, context?: unknown): DurableOperationRecord { if (!this.consumer) throw new ControlPlaneOwnershipError("AUTHORITY_REQUIRED", "cutover start requires trusted host authority"); const {snapshot,stateRoot,request,requestHash,operationId,subject} = planCutoverStart(this.config.stateDir,input); @@ -821,6 +924,7 @@ export class DurableOperationManager { recipe: input.recipe, frozenInputs: before, ...(input.ownerDirectIsolated === true ? { ownerDirectIsolated: true } : {}), + ...(input.ownerDirectExecution === true ? { ownerDirectExecution: true } : {}), }; const requestHash = hashJson(request); const operationId = stableOperationId("dependency_sync", workspaceRoot, input.attemptKey); @@ -829,15 +933,15 @@ export class DurableOperationManager { } async dependencySync(input: DependencySyncInput, consumerContext?: unknown): Promise { - const ownerDirectIsolated = input.ownerDirectIsolated === true; - if (!ownerDirectIsolated && !this.consumer) { - throw new ControlPlaneOwnershipError("AUTHORITY_REQUIRED", "dependency sync requires a trusted host authority reader"); + const ownerDirect = input.ownerDirectIsolated === true || input.ownerDirectExecution === true; + if (!ownerDirect && !this.consumer) { + throw new ControlPlaneOwnershipError("AUTHORITY_REQUIRED", "historical bound dependency sync requires trusted host authority from its original reader"); } const consumer = this.consumer; const {subject, request, workspaceRoot, frozenInputs, before, baseRevision, requestHash, operationId} = await this.planDependencySync(input); - const directWitnessId = "OWNER_DIRECT_ISOLATED"; + const directWitnessId = input.ownerDirectExecution === true ? "OWNER_DIRECT_CHECKOUT" : "OWNER_DIRECT_ISOLATED"; const prepared = this.store.atomic(() => { - if (ownerDirectIsolated) { + if (ownerDirect) { const value = this.store.createOrReplay({ operationId, attemptKey: input.attemptKey, @@ -866,7 +970,7 @@ export class DurableOperationManager { if (!created) return replayResult(record); const finish = (patch: Parameters[1]) => this.store.atomic(() => { - if (!ownerDirectIsolated) { + if (!ownerDirect) { consumer!.finish(consumerContext, subject, binding!, pinnedVersion!); } return this.store.finish(operationId, patch); @@ -875,7 +979,7 @@ export class DurableOperationManager { if (await readGitHead(workspaceRoot) !== baseRevision || hashJson(await hashFiles(workspaceRoot, frozenInputs)) !== hashJson(before)) { throw new Error("Frozen dependency input or base revision changed before launch"); } - if (!ownerDirectIsolated) { + if (!ownerDirect) { consumer!.assertPinned(consumerContext, subject, binding!, pinnedVersion!); } const command = dependencyCommand(input.recipe); @@ -886,7 +990,7 @@ export class DurableOperationManager { this.store.recordDependencyTerminal( operationId, requestHash, - ownerDirectIsolated ? directWitnessId : binding!.leaseId, + ownerDirect ? directWitnessId : binding!.leaseId, result.exitCode, frozenInputsUnchanged, ); @@ -929,16 +1033,18 @@ export class DurableOperationManager { if ( !record || record.kind !== "dependency_sync" || - record.request.ownerDirectIsolated !== true + record.request.ownerDirectIsolated !== true && + record.request.ownerDirectExecution !== true ) { - throw new ControlPlaneOwnershipError("AUTHORITY_REQUIRED", "owner-direct isolated reconciliation is not authorized for this operation"); + throw new ControlPlaneOwnershipError("AUTHORITY_REQUIRED", "owner-direct dependency reconciliation is not authorized for this operation"); } if (record.status === "succeeded" || record.status === "failed") return record; const witness = this.store.readDependencyTerminal(operationId); + const expectedWitnessId = record.request.ownerDirectExecution === true ? "OWNER_DIRECT_CHECKOUT" : "OWNER_DIRECT_ISOLATED"; if ( !witness || witness.requestHash !== record.requestHash || - witness.leaseId !== "OWNER_DIRECT_ISOLATED" + witness.leaseId !== expectedWitnessId ) { throw new DurableOperationError( "RECONCILIATION_REQUIRED", @@ -951,7 +1057,7 @@ export class DurableOperationManager { retrySafe: false, receipt: { reconciliation: { - mode: "OWNER_DIRECT_ISOLATED", + mode: expectedWitnessId, requestHash: witness.requestHash, exitCode: witness.exitCode, frozenInputsUnchanged: witness.frozenInputsUnchanged, @@ -985,10 +1091,14 @@ export class DurableOperationManager { async reconcile(operationId: string, consumerContext?: unknown): Promise { const record = this.store.getByOperationId(operationId); if (!record) throw new DurableOperationError("RECONCILIATION_REQUIRED", `Unknown durable operation: ${operationId}`); - if (record.kind === "cutover_start") return this.reconcileCutoverStart(operationId,consumerContext); + if (record.kind === "cutover_start") { + return record.request.ownerDirect === true + ? this.reconcileDirectCutoverStart(operationId) + : this.reconcileCutoverStart(operationId, consumerContext); + } if (record.kind === "git_push") return await this.reconcileGitPush(operationId); if (record.kind === "dependency_sync") { - if (record.request.ownerDirectIsolated === true) { + if (record.request.ownerDirectIsolated === true || record.request.ownerDirectExecution === true) { return this.reconcileOwnerDirectDependencySync(operationId); } if (!this.consumer || typeof record.request.baseRevision !== "string") throw new ControlPlaneOwnershipError("AUTHORITY_REQUIRED", "dependency reconciliation requires revision-bound host authority"); diff --git a/src/mcp-cutover.ts b/src/mcp-cutover.ts index 3b69235b7..39e41814e 100644 --- a/src/mcp-cutover.ts +++ b/src/mcp-cutover.ts @@ -6,6 +6,7 @@ import type { SelfRestartActuator } from "./cutover-restart.js"; import type { BuildReadyReceipt, CutoverCoordinationBinding, + CutoverDirectOperationBinding, CutoverBindingRepairReceipt, CutoverActivationBinding, CutoverDrainEvidence, @@ -193,12 +194,13 @@ export class McpCutoverController { private readonly now: () => number = Date.now, ) {} - begin(expectedNewIdentity: ExpectedCutoverIdentity, expiresAt?: string, coordinationBinding?: CutoverCoordinationBinding): DurableCutoverRecord { + begin(expectedNewIdentity: ExpectedCutoverIdentity, expiresAt?: string, coordinationBinding?: CutoverCoordinationBinding, directOperation?: CutoverDirectOperationBinding): DurableCutoverRecord { return this.store.begin({ oldServerIdentity: this.currentIdentity, expectedNewIdentity, expiresAt, coordinationBinding, + directOperation, }); } diff --git a/src/server.test.ts b/src/server.test.ts index 08bb7d079..42f7e6d02 100644 --- a/src/server.test.ts +++ b/src/server.test.ts @@ -1174,6 +1174,88 @@ test("cutover MCP control exposes bounded lease lifecycle and schedules self res } }); +test("C4 MCP direct cutover starts and reconciles without paired Carrier or coordination reader", async () => { + const root = await mkdtemp(join(tmpdir(), "devspace-c4-direct-cutover-mcp-")); + const stateDir = join(root, ".state"); + const config = loadConfig({ + DEVSPACE_CONFIG_DIR: join(root, ".config"), + DEVSPACE_ALLOWED_ROOTS: root, + DEVSPACE_STATE_DIR: stateDir, + DEVSPACE_OAUTH_OWNER_TOKEN: "test-owner-token-that-is-long-enough", + DEVSPACE_TOOL_MODE: "full", + PORT: "1", + }); + const workspaceStore = new SqliteWorkspaceStore(stateDir); + const manager = new DurableOperationManager(config); + const currentIdentity = { + serverInstanceId: "direct-old-server", + sourceCommit: "a".repeat(40), + buildId: "direct-old-build", + capabilityManifestSha256: "c".repeat(64), + }; + const controller = new McpCutoverController(new CutoverStateStore(stateDir), currentIdentity); + const server = createMcpServer( + config, + new WorkspaceRegistry(config, workspaceStore), + createReviewCheckpointManager(), + new ProcessSessionManager(), + () => [], + [], + undefined, + undefined, + undefined, + manager, + { + controller, + transportEvidence: () => ({ activeSessions: 0, oldestAgeMs: 0 }), + reconcileDurableState: async () => ({ + workspaceQueryable: true, + agentQueryable: true, + agentReconciled: true, + }), + }, + ); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "devspace-c4-direct", version: "1.0.0" }); + try { + await Promise.all([client.connect(clientTransport), server.connect(serverTransport)]); + const args = { + expectedSourceCommit: "b".repeat(40), + expectedBuildId: "direct-target-build", + expectedCapabilityManifestSha256: "c".repeat(64), + attemptKey: "c4-direct-mcp-cutover", + }; + const first = await client.callTool({ name: "cutover_start", arguments: args }); + assert.equal(first.isError, undefined, JSON.stringify(first)); + const firstData = structuredContent(first); + const active = firstData.cutover as { directOperation?: {operationId: string; requestHash: string}; coordinationBinding?: unknown }; + assert.equal(active.coordinationBinding, undefined); + assert.equal(active.directOperation?.operationId, firstData.operationId); + assert.deepEqual(manager.store.getByOperationId(firstData.operationId as string)?.request.ownerDirect, true); + const replay = await client.callTool({ name: "cutover_start", arguments: args }); + assert.equal(replay.isError, undefined, JSON.stringify(replay)); + assert.equal(structuredContent(replay).operationId, firstData.operationId); + const reconciled = await client.callTool({ + name: "operation_reconcile", + arguments: { operationId: firstData.operationId }, + }); + assert.equal(reconciled.isError, undefined, JSON.stringify(reconciled)); + assert.equal(structuredContent(reconciled).status, "succeeded"); + const conflict = await client.callTool({ + name: "cutover_start", + arguments: { ...args, expectedBuildId: "wrong-new-build" }, + }); + assert.equal(conflict.isError, true); + assert.equal(controller.record()?.cutoverId, (firstData.cutover as {cutoverId: string}).cutoverId); + } finally { + await client.close(); + await server.close(); + manager.close(); + workspaceStore.close(); + await rm(root, { recursive: true, force: true }); + } +}); + test("cutover restart tool is absent when no bounded self actuator is available", async () => { const root = await mkdtemp(join(tmpdir(), "devspace-cutover-no-actuator-")); const stateDir = join(root, ".state"); @@ -3230,7 +3312,7 @@ test("OWNER_DIRECT workspace_clone works and dependency_sync denies unauthentica }); -test("OWNER_DIRECT dependency_sync bypasses carrier only for a managed isolated worktree", async (t) => { +test("C4 OWNER_DIRECT dependency_sync supports managed isolated worktree and single-conversation checkout", async (t) => { const context = await fixture(t, { git: true }); await writeFile(join(context.project, "package.json"), JSON.stringify({ name: "isolated-fixture", version: "1.0.0" }) + "\n"); await writeFile(join(context.project, "package-lock.json"), JSON.stringify({ @@ -3272,8 +3354,35 @@ test("OWNER_DIRECT dependency_sync bypasses carrier only for a managed isolated }, _meta: { "openai/session": "checkout-dependency-sync" }, }); - assert.equal(checkoutResult.isError, true); - assert.match(JSON.stringify(checkoutResult), /authenticated MCP client context is required/); + assert.equal(checkoutResult.isError, undefined, JSON.stringify(checkoutResult)); + assert.equal(structuredContent(checkoutResult).status, "succeeded"); + assert.equal( + ((structuredContent(checkoutResult).request as Record) ?? {}).ownerDirectExecution, + true, + ); + const checkoutReplay = await context.client.callTool({ + name: "dependency_sync", + arguments: { + workspaceId: checkoutWorkspaceId, + attemptKey: "checkout-owner-direct-deps", + recipe: "npm_ci", + }, + _meta: { "openai/session": "checkout-dependency-sync" }, + }); + assert.equal(structuredContent(checkoutReplay).operationId, structuredContent(checkoutResult).operationId); + + await callOpen(context.client, context.project, "competing-dependency-sync", "checkout"); + const shared = await context.client.callTool({ + name: "dependency_sync", + arguments: { + workspaceId: checkoutWorkspaceId, + attemptKey: "checkout-shared-must-not-run", + recipe: "npm_ci", + }, + _meta: { "openai/session": "checkout-dependency-sync" }, + }); + assert.equal(shared.isError, true); + assert.match(JSON.stringify(shared), /CONVERSATION_CHECKOUT_SHARED|managed worktree|shared by active conversations/i); }); test("command_status metadata annotations and minimal mode visibility", async (t) => { @@ -4242,11 +4351,14 @@ test("#386: real server /mcp keeps unrelated workspace mutation usable during dr assert.equal(startRes.status, 200); const startText=await startRes.text(); const startData=startText.split("\n").find(line=>line.startsWith("data: ")); - assert.equal(JSON.parse(startData?startData.slice(6):startText).result.isError,true); - assert.equal(new CutoverStateStore(stateDir).get(),undefined); - // Arrange drain through the trusted fixture API; this test covers reconnect. - const runtime=await (await fetch(`http://127.0.0.1:${port}/identity`)).json() as any; - new CutoverStateStore(stateDir).begin({oldServerIdentity:{serverInstanceId:runtime.serverInstanceId,sourceCommit:runtime.sourceCommit,buildId:runtime.buildId,capabilityManifestSha256:runtime.capabilityManifest.manifestSha256},expectedNewIdentity:{sourceCommit:"a".repeat(40),buildId:"build-p04"}}); + const startResult=JSON.parse(startData?startData.slice(6):startText).result; + assert.equal(startResult.isError,undefined, JSON.stringify(startResult)); + const startedCutover=new CutoverStateStore(stateDir).get(); + assert.equal(startedCutover?.phase,"prepared"); + assert.equal(startedCutover?.coordinationBinding,undefined); + assert.equal(startedCutover?.directOperation?.operationId,startResult.structuredContent?.operationId); + // The actual MCP direct start is now the fixture's persisted cutover. + // Reconnect and drain must operate on that exact generation. // 3. Client 2 (reconnecting ChatGPT connector after drop, starting new session during drain) diff --git a/src/server.ts b/src/server.ts index e45894d64..f3b68708d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -114,6 +114,7 @@ import type { WorkspaceSession } from "./workspace-store.js"; import { ProcessSessionManager, type ProcessSnapshot } from "./process-sessions.js"; import { DurableOperationManager, + DurableOperationStore, planCutoverStart, DurableOperationError, type DurableOperationRecord, @@ -1032,7 +1033,6 @@ function registerCodexProcessTools( config: ServerConfig, workspaces: WorkspaceRegistry, processSessions: ProcessSessionManager, - carrierBindings?: CarrierBindingStore, ): void { registerAppTool( server, @@ -2142,9 +2142,8 @@ function registerCutoverMcpTools( _meta: {}, annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: false }, }, - async ({ expectedSourceCommit, expectedBuildId, expectedCapabilityManifestSha256, expiresAt, attemptKey }, extra) => { - const context = dependencyConsumerContext(extra); - if (!durableOperations) throw new ControlPlaneOwnershipError("AUTHORITY_REQUIRED", "cutover start requires trusted host coordination"); + async ({ expectedSourceCommit, expectedBuildId, expectedCapabilityManifestSha256, expiresAt, attemptKey }) => { + if (!durableOperations) throw new CutoverStateError("Durable cutover manager is unavailable."); if (expectedCapabilityManifestSha256 && control.probeBuildReady) { const probe = await control.probeBuildReady({ sourceCommit: expectedSourceCommit, @@ -2167,7 +2166,7 @@ function registerCutoverMcpTools( }; const currentIdentity = control.controller.currentIdentity; const defaultAttempt = createHash("sha256").update(JSON.stringify({currentIdentity,expectedIdentity,expiresAt})).digest("hex"); - const operation = durableOperations.startCutover({attemptKey:attemptKey ?? defaultAttempt,currentIdentity,expectedIdentity,expiresAt},context); + const operation = durableOperations.startDirectCutover({attemptKey:attemptKey ?? defaultAttempt,currentIdentity,expectedIdentity,expiresAt}); if (operation.status !== "succeeded") throw new CutoverStateError(`Cutover operation ${operation.operationId} requires exact operation_reconcile; no new start is authorized.`); const record = control.controller.record(); if (!record || record.cutoverId !== operation.receipt?.cutoverId) throw new CutoverStateError(`Cutover operation ${operation.operationId} readback changed; reconciliation required.`); @@ -3990,7 +3989,15 @@ export function createMcpServer( workspace.mode === "worktree" && workspace.worktree?.managed === true && safety.state === "ISOLATED_WORKTREE"; - const consumerContext = ownerDirectIsolated ? undefined : dependencyConsumerContext(extra); + const ownerDirectExecution = + workspace.mode === "checkout" && + safety.state === "SINGLE_CONVERSATION_CHECKOUT"; + if (!ownerDirectIsolated && !ownerDirectExecution) { + throw new ControlPlaneOwnershipError( + "AUTHORITY_REQUIRED", + "An authenticated MCP client context is required for single-conversation checkout dependency sync; use an isolated managed worktree otherwise.", + ); + } try { const operation = await durableOperations.dependencySync({ workspaceId, @@ -3998,7 +4005,8 @@ export function createMcpServer( attemptKey, recipe, ownerDirectIsolated, - }, consumerContext); + ownerDirectExecution, + }); return operationResponse(operation); } catch (error) { if (error instanceof DurableOperationError && error.operation) { @@ -4046,11 +4054,12 @@ export function createMcpServer( }, async ({ operationId }, extra) => { const record = durableOperations.store.getByOperationId(operationId); - const context = - record?.kind === "cutover_start" || - (record?.kind === "dependency_sync" && record.request.ownerDirectIsolated !== true) - ? dependencyConsumerContext(extra) - : undefined; + const historicalBoundOperation = + (record?.kind === "cutover_start" && record.request.ownerDirect !== true) || + (record?.kind === "dependency_sync" && + record.request.ownerDirectIsolated !== true && + record.request.ownerDirectExecution !== true); + const context = historicalBoundOperation ? dependencyConsumerContext(extra) : undefined; return operationResponse(await durableOperations.reconcile(operationId, context)); }, ); @@ -6795,6 +6804,21 @@ function requireManifestBoundServerInstanceId(value: string | undefined): string return normalized; } +/** + * Legacy Carrier authority is opened only for exact persisted bound effects. + * New cutovers and OWNER_DIRECT dependency operations never create Carrier + * governance, and historical records keep their original recovery reader. + */ +function requiresLegacyCarrierRecovery(stateDir: string): boolean { + if (new CutoverStateStore(stateDir).get()?.coordinationBinding) return true; + const operations = new DurableOperationStore(stateDir); + try { + return operations.hasLegacyBoundOperations(); + } finally { + operations.close(); + } +} + export function createServer( config = loadConfig(), options: CreateServerOptions = {}, @@ -6859,7 +6883,10 @@ export function createServer( const reviewCheckpoints = createReviewCheckpointManager(); const processSessions = new ProcessSessionManager(); initializationCleanups.push(() => processSessions.shutdown()); - const carrierBindings = options.coordination ? undefined : new CarrierBindingStore(config.stateDir, options.carrierClock, options.completionBindings); + const carrierBindings = options.coordination || + (!options.completionBindings?.length && !requiresLegacyCarrierRecovery(config.stateDir)) + ? undefined + : new CarrierBindingStore(config.stateDir, options.carrierClock, options.completionBindings); if (carrierBindings) initializationCleanups.push(() => carrierBindings.close()); const durableOperations = new DurableOperationManager(config, undefined, options.coordination ?? carrierBindings?.readers); initializationCleanups.push(() => durableOperations.close()); From f954a20cad6e041ef3f689573f031c4d5a6b4b8e Mon Sep 17 00:00:00 2001 From: James3014 <185240413+James3014@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:45:38 +0800 Subject: [PATCH 2/4] fix(cutover): preserve pending legacy Carrier terminalization before direct start (#471) --- src/durable-operations.test.ts | 64 ++++++++++++++++++++++++++++++++++ src/durable-operations.ts | 32 +++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/src/durable-operations.test.ts b/src/durable-operations.test.ts index 28cbb4f67..e3233b0ff 100644 --- a/src/durable-operations.test.ts +++ b/src/durable-operations.test.ts @@ -17,6 +17,7 @@ import { DurableOperationError, DurableOperationManager, DurableOperationStore, + cutoverTerminalRecordHash, planCutoverStart, hashJson, type CommandRunner, @@ -483,6 +484,69 @@ test("C4 refuses to downgrade a persisted Carrier-bound cutover into new OWNER_D } }); +test("C4 cannot archive a closed Carrier generation while its durable lease terminalization is incomplete", async () => { + const f = await fixture(); + const manager = new DurableOperationManager(f.config); + try { + const old = { + attemptKey: "legacy-closed-awaiting-ack", + currentIdentity: {serverInstanceId: "bound-owner", sourceCommit: "a".repeat(40), buildId: "old"}, + expectedIdentity: {sourceCommit: "b".repeat(40), buildId: "old-target"}, + }; + const plan = planCutoverStart(f.config.stateDir, old); + const coordinationBinding = { + leaseId: "lease-legacy-closed", + pinnedLeaseVersion: 3, + operationHandle: plan.operationId, + requestHash: plan.requestHash, + ownerThread: "legacy-controller", + }; + const bound = manager.store.createOrReplay({ + operationId: plan.operationId, attemptKey: old.attemptKey, + requestHash: plan.requestHash, kind: "cutover_start", + scopeRoot: plan.stateRoot, request: {...plan.request, coordinationBinding}, + }); + assert.equal(bound.record.status, "started"); + const state = new CutoverStateStore(f.config.stateDir); + const existing = state.begin({ + oldServerIdentity: old.currentIdentity, + expectedNewIdentity: old.expectedIdentity, + coordinationBinding, + }); + state.close(existing.cutoverId, { + closedByServerInstanceId: "replacement-bound", workspaceQueryable: true, + agentQueryable: true, agentReconciled: true, reconciledAt: new Date().toISOString(), + }); + assert.equal(state.get()?.phase, "closed"); + const next = { + attemptKey: "c4-new-after-terminal", + currentIdentity: {serverInstanceId: "replacement-bound", sourceCommit: "b".repeat(40), buildId: "old-target"}, + expectedIdentity: {sourceCommit: "c".repeat(40), buildId: "new-target"}, + }; + assert.throws(() => manager.startDirectCutover(next), /bound.*terminal|legacy.*terminal|lifecycle.*terminal/i); + assert.equal(state.get()?.cutoverId, existing.cutoverId, "legacy recovery must remain physically addressable"); + const nextId = planCutoverStart(f.config.stateDir, next).operationId; + assert.equal(manager.store.getByOperationId(nextId), undefined, "no new intent may be poisoned before legacy terminalization"); + manager.store.finish(plan.operationId, { + status: "succeeded", retrySafe: false, + receipt: {cutoverId: existing.cutoverId, lifecycleTerminal: true, terminalRecordHash: "0".repeat(64)}, + }); + assert.throws(() => manager.startDirectCutover(next), /bound.*terminal|legacy.*terminal|digest|hash/i); + assert.equal(state.get()?.cutoverId, existing.cutoverId); + manager.store.finish(plan.operationId, { + status: "succeeded", retrySafe: false, + receipt: {cutoverId: existing.cutoverId, lifecycleTerminal: true, terminalRecordHash: cutoverTerminalRecordHash(state.get()!)}, + }); + const after = manager.startDirectCutover(next); + assert.equal(after.status, "succeeded"); + assert.equal(state.get()?.directOperation?.operationId, after.operationId); + assert.notEqual(state.get()?.cutoverId, existing.cutoverId); + } finally { + manager.close(); + await f.cleanup(); + } +}); + test("C4 direct cutover lost-ACK reconciles exact intent, never recreates absent or mismatched physical effects", async () => { const f = await fixture(); const manager = new DurableOperationManager(f.config); diff --git a/src/durable-operations.ts b/src/durable-operations.ts index c4bfca55a..af4899401 100644 --- a/src/durable-operations.ts +++ b/src/durable-operations.ts @@ -609,6 +609,32 @@ export class DurableOperationManager { }); } + /** + * A closed Carrier-bound cutover can still have an unfinished SQLite/lease + * terminalization after its file is closed. Archiving that file prematurely + * would strand the original generation and make bound recovery impossible. + */ + private assertLegacyBoundCutoverIsTerminal(stateRoot: string): void { + const existing = new CutoverStateStore(stateRoot).get(); + if (!existing?.coordinationBinding || existing.phase !== "closed") return; + const binding = existing.coordinationBinding; + const operation = this.store.getByOperationId(binding.operationHandle); + if (!operation || operation.kind !== "cutover_start" || + operation.scopeRoot !== stateRoot || + operation.requestHash !== binding.requestHash || + !isDeepStrictEqual(operation.request.coordinationBinding, binding) || + operation.receipt?.cutoverId !== existing.cutoverId || + operation.receipt?.lifecycleTerminal !== true || + operation.receipt?.terminalRecordHash !== cutoverTerminalRecordHash(existing) || + (operation.status !== "succeeded" && operation.status !== "failed")) { + throw new DurableOperationError( + "RECONCILIATION_REQUIRED", + "Closed legacy Carrier-bound cutover cannot be archived until its exact terminal operation and lease are durable.", + operation, + ); + } + } + /** * New cutovers are DevSpace-native effects, not Carrier-governed actions. * The intent is durably recorded before the cutover file is written; the @@ -619,6 +645,9 @@ export class DurableOperationManager { const { snapshot, stateRoot, operationId } = planned; const request = { ...planned.request, ownerDirect: true }; const requestHash = hashJson(request); + // This is a pre-intent admission gate: known historical recovery blockers + // must not poison a new direct attempt with an outcome_unknown record. + this.assertLegacyBoundCutoverIsTerminal(stateRoot); const intent = this.store.createOrReplay({ operationId, attemptKey: snapshot.attemptKey, @@ -633,6 +662,9 @@ export class DurableOperationManager { if (!isDeepStrictEqual(this.store.getByOperationId(operationId), intent.record)) { throw new ControlPlaneOwnershipError("CAS_CONFLICT", "direct cutover intent changed before file write"); } + // Recheck under the durable transaction immediately before changing + // the separate cutover file, including late terminal receipt changes. + this.assertLegacyBoundCutoverIsTerminal(stateRoot); const controller = new McpCutoverController(new CutoverStateStore(stateRoot), snapshot.currentIdentity); controller.begin(snapshot.expectedIdentity, snapshot.expiresAt, undefined, { operationId, requestHash }); return this.reconcileDirectCutoverStart(operationId); From af27f167b225bc119115fb1034ca1cb2ce79ec10 Mon Sep 17 00:00:00 2001 From: James3014 <185240413+James3014@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:40:50 +0800 Subject: [PATCH 3/4] fix(cutover): reject known active fence before recording a new direct attempt (#471) --- src/durable-operations.test.ts | 44 +++++++++++++++++++++++++++++--- src/durable-operations.ts | 46 +++++++++++++++++++++------------- 2 files changed, 69 insertions(+), 21 deletions(-) diff --git a/src/durable-operations.test.ts b/src/durable-operations.test.ts index e3233b0ff..136997c6d 100644 --- a/src/durable-operations.test.ts +++ b/src/durable-operations.test.ts @@ -471,13 +471,49 @@ test("C4 refuses to downgrade a persisted Carrier-bound cutover into new OWNER_D expectedNewIdentity: input.expectedIdentity, coordinationBinding, }); - const attempt = manager.startDirectCutover(input); - assert.equal(attempt.status, "outcome_unknown"); - assert.equal(attempt.retrySafe, false); + const operationId = planCutoverStart(f.config.stateDir, input).operationId; + assert.throws(() => manager.startDirectCutover(input), /Unresolved cutover.*fence/i); + assert.equal(manager.store.getByOperationId(operationId), undefined, "no intent may be persisted for a known active legacy fence"); assert.deepEqual(new CutoverStateStore(f.config.stateDir).get()?.coordinationBinding, coordinationBinding); assert.equal(new CutoverStateStore(f.config.stateDir).get()?.directOperation, undefined); assert.equal(new CutoverStateStore(f.config.stateDir).get()?.cutoverId, bound.cutoverId); - assert.throws(() => manager.reconcileDirectCutoverStart(attempt.operationId), /no retry is authorized/); + } finally { + manager.close(); + await f.cleanup(); + } +}); + +test("C4 occupied native cutover fence denies pre-intent and permits same attemptKey after genuine closure", async () => { + const f = await fixture(); + const manager = new DurableOperationManager(f.config); + try { + const state = new CutoverStateStore(f.config.stateDir); + const first = state.begin({ + oldServerIdentity: {serverInstanceId: "old-active", sourceCommit: "a".repeat(40), buildId: "build-a"}, + expectedNewIdentity: {sourceCommit: "b".repeat(40), buildId: "build-b"}, + }); + const input = { + attemptKey: "c4-known-fence-retry", + currentIdentity: {serverInstanceId: "new-active", sourceCommit: "b".repeat(40), buildId: "build-b"}, + expectedIdentity: {sourceCommit: "c".repeat(40), buildId: "build-c"}, + }; + const operationId = planCutoverStart(f.config.stateDir, input).operationId; + assert.throws(() => manager.startDirectCutover(input), /Unresolved cutover.*fence/i); + assert.equal(manager.store.getByOperationId(operationId), undefined, "pre-intent collision must leave the attempt key reusable"); + assert.equal(state.get()?.cutoverId, first.cutoverId); + state.close(first.cutoverId, { + closedByServerInstanceId: "closed-active", + workspaceQueryable: true, + agentQueryable: true, + agentReconciled: true, + reconciledAt: new Date().toISOString(), + }); + const started = manager.startDirectCutover(input); + assert.equal(started.status, "succeeded"); + assert.equal(started.operationId, operationId); + assert.deepEqual(manager.startDirectCutover(input), started, "same-effect replay may not be rejected by its own active fence"); + assert.equal(state.get()?.directOperation?.operationId, operationId); + assert.notEqual(state.get()?.cutoverId, first.cutoverId); } finally { manager.close(); await f.cleanup(); diff --git a/src/durable-operations.ts b/src/durable-operations.ts index af4899401..c3f166653 100644 --- a/src/durable-operations.ts +++ b/src/durable-operations.ts @@ -3,7 +3,7 @@ import type { SelfRestartActuator } from "./cutover-restart.js"; import type { CompletionSelection } from "./current-completion-matrix.js"; import { isDeepStrictEqual } from "node:util"; import { McpCutoverController, compareServerIdentity, type DurableReconciliationWitness } from "./mcp-cutover.js"; -import { CutoverStateStore, type CutoverServerIdentity, type CutoverDrainEvidence, type BuildReadyReceipt, type ExpectedCutoverIdentity, type CutoverCoordinationBinding, type CutoverActivationBinding } from "./cutover-state.js"; +import { CutoverStateError, CutoverStateStore, type CutoverServerIdentity, type CutoverDrainEvidence, type BuildReadyReceipt, type ExpectedCutoverIdentity, type CutoverCoordinationBinding, type CutoverActivationBinding } from "./cutover-state.js"; import { ControlPlaneConsumer, type ControlPlaneConsumerOptions, type DependencyReconciliationEvidence } from "./control-plane-consumer.js"; import { ControlPlaneOwnershipError, ControlPlaneOwnershipStore, type HandoffInput, type TakeoverInput, type ReconciliationReceipt } from "./control-plane-ownership.js"; import { createHash } from "node:crypto"; @@ -610,13 +610,18 @@ export class DurableOperationManager { } /** - * A closed Carrier-bound cutover can still have an unfinished SQLite/lease - * terminalization after its file is closed. Archiving that file prematurely - * would strand the original generation and make bound recovery impossible. + * Known occupied generations must reject a new cutover before its durable + * attempt is created. A closed Carrier-bound generation also needs an exact + * terminal receipt before its file can be archived for the next generation. */ - private assertLegacyBoundCutoverIsTerminal(stateRoot: string): void { + private assertCutoverFenceAvailable(stateRoot: string): void { const existing = new CutoverStateStore(stateRoot).get(); - if (!existing?.coordinationBinding || existing.phase !== "closed") return; + if (existing && existing.phase !== "closed") { + throw new CutoverStateError( + `Unresolved cutover ${existing.cutoverId} already owns the durable cutover fence.`, + ); + } + if (!existing?.coordinationBinding) return; const binding = existing.coordinationBinding; const operation = this.store.getByOperationId(binding.operationHandle); if (!operation || operation.kind !== "cutover_start" || @@ -645,16 +650,22 @@ export class DurableOperationManager { const { snapshot, stateRoot, operationId } = planned; const request = { ...planned.request, ownerDirect: true }; const requestHash = hashJson(request); - // This is a pre-intent admission gate: known historical recovery blockers - // must not poison a new direct attempt with an outcome_unknown record. - this.assertLegacyBoundCutoverIsTerminal(stateRoot); - const intent = this.store.createOrReplay({ - operationId, - attemptKey: snapshot.attemptKey, - requestHash, - kind: "cutover_start", - scopeRoot: stateRoot, - request, + // Admit a genuinely new attempt only if no older generation owns the + // physical cutover fence. An already-persisted attempt bypasses this + // preflight so that exact lost-ACK replay can still reconcile its own + // active generation without ever starting another effect. + const intent = this.store.atomic(() => { + if (!this.store.getByAttempt(stateRoot, snapshot.attemptKey)) { + this.assertCutoverFenceAvailable(stateRoot); + } + return this.store.createOrReplay({ + operationId, + attemptKey: snapshot.attemptKey, + requestHash, + kind: "cutover_start", + scopeRoot: stateRoot, + request, + }); }); if (!intent.created) return this.reconcileDirectCutoverStart(operationId); try { @@ -664,7 +675,8 @@ export class DurableOperationManager { } // Recheck under the durable transaction immediately before changing // the separate cutover file, including late terminal receipt changes. - this.assertLegacyBoundCutoverIsTerminal(stateRoot); + // A genuinely racing effect may still be unknown; never blind retry it. + this.assertCutoverFenceAvailable(stateRoot); const controller = new McpCutoverController(new CutoverStateStore(stateRoot), snapshot.currentIdentity); controller.begin(snapshot.expectedIdentity, snapshot.expiresAt, undefined, { operationId, requestHash }); return this.reconcileDirectCutoverStart(operationId); From 1b606ca97f6aa91a7e57532ed4166d35cd7af116 Mon Sep 17 00:00:00 2001 From: James3014 <185240413+James3014@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:48:15 +0800 Subject: [PATCH 4/4] fix(cutover): require terminal historical lease before archiving C4 generation (#471) --- src/durable-operations.test.ts | 43 +++++++++++++++++++++++++++++ src/durable-operations.ts | 49 +++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/durable-operations.test.ts b/src/durable-operations.test.ts index 136997c6d..49ab9861a 100644 --- a/src/durable-operations.test.ts +++ b/src/durable-operations.test.ts @@ -12,6 +12,8 @@ import type { ControlPlaneOwnershipStore } from "./control-plane-ownership.js"; import type { ControlPlaneConsumerOptions } from "./control-plane-consumer.js"; import { loadConfig } from "./config.js"; import { CutoverStateStore } from "./cutover-state.js"; +import { openDatabase } from "./db/client.js"; +import { initializeControlPlaneOwnershipDatabase } from "./control-plane-ownership.js"; import { canonicalizePath } from "./roots.js"; import { DurableOperationError, @@ -573,6 +575,47 @@ test("C4 cannot archive a closed Carrier generation while its durable lease term status: "succeeded", retrySafe: false, receipt: {cutoverId: existing.cutoverId, lifecycleTerminal: true, terminalRecordHash: cutoverTerminalRecordHash(state.get()!)}, }); + assert.throws(() => manager.startDirectCutover(next), /legacy.*lease.*terminal|terminal.*lease/i, + "a matching operation receipt cannot waive the original Carrier lease release"); + assert.equal(state.get()?.cutoverId, existing.cutoverId); + + const db = openDatabase(f.config.stateDir); + try { + initializeControlPlaneOwnershipDatabase(db.sqlite); + const now = new Date().toISOString(); + db.sqlite.prepare(`insert into control_plane_resource_leases ( + lease_id,repository_key,resource_kind,resource_id,resource,operation,scope_json,base_revision, + idempotency_key,owner_thread,grant_json,grant_version,version,terminal_state,expires_at, + created_at,updated_at,active_operation_handle,operation_state + ) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`).run( + coordinationBinding.leaseId, "james3014/devspace", "filesystem", plan.stateRoot, plan.stateRoot, + "cutover_start", JSON.stringify([plan.stateRoot]), old.currentIdentity.sourceCommit, + old.attemptKey, coordinationBinding.ownerThread, + JSON.stringify({ + repository: "james3014/devspace", goal: "legacy-closure-fixture", + coordinatorThread: coordinationBinding.ownerThread, evidenceHash: "fixture", + }), + 1, 3, null, "2100-01-01T00:00:00.000Z", now, now, null, "finished", + ); + assert.throws(() => manager.startDirectCutover(next), /legacy.*lease.*terminal|terminal.*lease/i, + "a finished operation with an unreleased state-root lease must remain recoverable"); + db.sqlite.prepare("update control_plane_resource_leases set terminal_state='released', version=version+1 where lease_id=?") + .run(coordinationBinding.leaseId); + db.sqlite.prepare("update control_plane_resource_leases set owner_thread='wrong-owner' where lease_id=?") + .run(coordinationBinding.leaseId); + assert.throws(() => manager.startDirectCutover(next), /legacy.*lease.*terminal|terminal.*lease/i, + "released lease owned by another historical principal cannot authorize archival"); + db.sqlite.prepare("update control_plane_resource_leases set owner_thread=? where lease_id=?") + .run(coordinationBinding.ownerThread, coordinationBinding.leaseId); + db.sqlite.prepare("update control_plane_resource_leases set resource='/wrong-resource' where lease_id=?") + .run(coordinationBinding.leaseId); + assert.throws(() => manager.startDirectCutover(next), /legacy.*lease.*terminal|terminal.*lease/i, + "released lease for another resource cannot authorize archival"); + db.sqlite.prepare("update control_plane_resource_leases set resource=? where lease_id=?") + .run(plan.stateRoot, coordinationBinding.leaseId); + } finally { + db.close(); + } const after = manager.startDirectCutover(next); assert.equal(after.status, "succeeded"); assert.equal(state.get()?.directOperation?.operationId, after.operationId); diff --git a/src/durable-operations.ts b/src/durable-operations.ts index c3f166653..41c5f2718 100644 --- a/src/durable-operations.ts +++ b/src/durable-operations.ts @@ -124,6 +124,33 @@ export class DurableOperationStore { this.database.close(); } + /** + * Read-only historical Carrier lease projection. An OWNER_DIRECT store may + * never have initialized Carrier tables, so absence must fail closed without + * creating schema (especially while an atomic cutover preflight is running). + */ + readHistoricalLease(leaseId: string) { + const table = this.database.sqlite.prepare( + "select 1 from sqlite_master where type='table' and name='control_plane_resource_leases'", + ).get(); + if (!table) return undefined; + return this.database.sqlite.prepare(` + select lease_id, owner_thread, operation, base_revision, resource, scope_json, + terminal_state, operation_state, active_operation_handle + from control_plane_resource_leases where lease_id=? limit 1 + `).get(leaseId) as { + lease_id: string; + owner_thread: string; + operation: string; + base_revision: string; + resource: string; + scope_json: string; + terminal_state: string | null; + operation_state: string | null; + active_operation_handle: string | null; + } | undefined; + } + createOwnershipStore(options: ControlPlaneConsumerOptions): ControlPlaneOwnershipStore { return new ControlPlaneOwnershipStore(this.database.sqlite, { ...options, @@ -634,7 +661,27 @@ export class DurableOperationManager { (operation.status !== "succeeded" && operation.status !== "failed")) { throw new DurableOperationError( "RECONCILIATION_REQUIRED", - "Closed legacy Carrier-bound cutover cannot be archived until its exact terminal operation and lease are durable.", + "Closed legacy Carrier-bound cutover cannot be archived until its exact terminal operation is durable.", + operation, + ); + } + // The old Carrier finish on deployed releases could record the terminal + // operation without releasing the state-root lease. Archiving its only + // active journal would strand the legacy release-terminal-lease recovery. + const lease = this.store.readHistoricalLease(binding.leaseId); + if (!lease || + lease.lease_id !== binding.leaseId || + !["released", "expired_reconciled"].includes(lease.terminal_state ?? "") || + lease.operation_state !== "finished" || + lease.active_operation_handle !== null || + lease.owner_thread !== binding.ownerThread || + lease.operation !== "cutover_start" || + lease.base_revision !== operation.request.baseRevision || + lease.resource !== stateRoot || + lease.scope_json !== JSON.stringify([stateRoot])) { + throw new DurableOperationError( + "RECONCILIATION_REQUIRED", + "Closed legacy Carrier lease is not terminal; release or reconcile its exact lease before a new direct cutover.", operation, ); }