From be85886cb86ec7d695fdd7d0bde7f62d7deac5cb Mon Sep 17 00:00:00 2001 From: James3014 <185240413+James3014@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:31:32 +0800 Subject: [PATCH 1/3] fix(control-plane): owner-local orphan lease hygiene and stale-path overlap robustness; sidecar lease-aware closed mode (#415, #516) Co-Authored-By: Claude Opus 5.5 --- scripts/ops/m5-cutover/m5-cutover-lib.mjs | 11 +++- scripts/ops/m5-cutover/m5-cutover-sidecar.mjs | 11 +++- scripts/ops/m5-cutover/m5-cutover.test.mjs | 13 +++++ src/carrier-binding.test.ts | 54 +++++++++++++++++++ src/control-plane-ownership.test.ts | 20 +++++++ src/control-plane-ownership.ts | 10 +++- 6 files changed, 116 insertions(+), 3 deletions(-) diff --git a/scripts/ops/m5-cutover/m5-cutover-lib.mjs b/scripts/ops/m5-cutover/m5-cutover-lib.mjs index 106442482..a85f4550c 100644 --- a/scripts/ops/m5-cutover/m5-cutover-lib.mjs +++ b/scripts/ops/m5-cutover/m5-cutover-lib.mjs @@ -72,7 +72,16 @@ export function recoveryFor(file, liveIsOld, liveIsTarget) { * Decide the sidecar mode from persisted state + live identity (read-only). * Returns "full" | "resume_drain" | "resume_restart" | "resume_finish"; throws Refuse otherwise. */ -export function selectMode({ file, contract, liveIsOld, liveIsTarget }) { +export function selectMode({ file, contract, liveIsOld, liveIsTarget, closedLease }) { + // `closedLease` is the state-root lease row of a CLOSED cutover ({leaseId, version, terminalState}) or null/undefined + // when unknown/absent. The file and the SQLite lease are separate stores: a crash between file close and lease + // finalization leaves a closed file with a still-held lease, and a new attempt would only fail on overlap (#516). + if (file && file.phase === "closed" && closedLease && !closedLease.terminalState) { + const id = file.cutoverId; + throw new Refuse(`cutover ${id} is closed but its state-root lease ${closedLease.leaseId} is not terminal; a new attempt would fail with an overlapping resource scope`, + `finish lease replay, do NOT start a new attempt: ${RUNBOOK_CLI} cutover release-terminal-lease --cutover-id ${id} --lease-id ${closedLease.leaseId} --lease-version ${closedLease.version} --carrier --carrier-version --terminal-record-hash --confirm ${id}`, + { activePhase: "closed", leaseId: closedLease.leaseId }); + } if (!file || file.phase === "closed") return "full"; // Branch a: brand-new attempt. const target = contract.cutover.expectedIdentity; const ours = sameDeep(file.oldServerIdentity, contract.cutover.currentIdentity) diff --git a/scripts/ops/m5-cutover/m5-cutover-sidecar.mjs b/scripts/ops/m5-cutover/m5-cutover-sidecar.mjs index 6775d7a77..f27e2d629 100755 --- a/scripts/ops/m5-cutover/m5-cutover-sidecar.mjs +++ b/scripts/ops/m5-cutover/m5-cutover-sidecar.mjs @@ -98,7 +98,16 @@ try { const liveIsTarget = matchesTarget(before, target) && before.mcp.serverInstanceId !== contract.cutover.currentIdentity.serverInstanceId; if (file && file.phase !== "closed") cutoverId = file.cutoverId; - const mode = selectMode({ file, contract, liveIsOld, liveIsTarget }); // "full" | "resume_drain" | "resume_restart" | "resume_finish" + // Closed file but state-root lease not yet terminal (crash between file close and lease finalization): read-only probe. + let closedLease; + if (file?.phase === "closed" && file.coordinationBinding?.leaseId) { + const probeBindings = new CarrierBindingStore(config.stateDir); + try { + const row = probeBindings.ownership.get(file.coordinationBinding.leaseId); + if (row) closedLease = { leaseId: row.leaseId, version: row.version, terminalState: row.terminalState ?? null }; + } finally { probeBindings.close(); } + } + const mode = selectMode({ file, contract, liveIsOld, liveIsTarget, closedLease }); // "full" | "resume_drain" | "resume_restart" | "resume_finish" log("phase_detected", { mode, activePhase: file?.phase ?? null, cutoverId: file?.cutoverId ?? null, liveIsOld, liveIsTarget }); if (mode !== "resume_finish" && Date.parse(contract.cutover.expiresAt) <= Date.now()) { diff --git a/scripts/ops/m5-cutover/m5-cutover.test.mjs b/scripts/ops/m5-cutover/m5-cutover.test.mjs index c88dc4812..8c8d8feb8 100644 --- a/scripts/ops/m5-cutover/m5-cutover.test.mjs +++ b/scripts/ops/m5-cutover/m5-cutover.test.mjs @@ -54,6 +54,19 @@ test("no active generation or a closed one selects full", () => { assert.equal(select(record({ phase: "closed" }), false, true), "full"); }); +test("closed file whose state-root lease is still held is refused with a release-terminal-lease hint (#516)", () => { + const closed = record({ phase: "closed", coordinationBinding: { leaseId: "lease_held" } }); + const held = { leaseId: "lease_held", version: 3, terminalState: null }; + const run = () => selectMode({ file: closed, contract, liveIsOld: false, liveIsTarget: true, closedLease: held }); + assert.throws(run, (error) => error instanceof Refuse + && /release-terminal-lease/.test(error.recovery) && /lease_held/.test(error.recovery) && /cut_1/.test(error.recovery) + && error.extra.activePhase === "closed" && error.extra.leaseId === "lease_held"); + // Liveness only: a terminal lease, an absent lease row, or no lease info keep the brand-new-attempt path. + for (const closedLease of [{ leaseId: "lease_held", version: 4, terminalState: "released" }, null, undefined]) { + assert.equal(selectMode({ file: closed, contract, liveIsOld: true, liveIsTarget: false, closedLease }), "full"); + } +}); + test("prepared without progress and old live resumes at drain", () => { assert.equal(select(record(), true, false), "resume_drain"); }); diff --git a/src/carrier-binding.test.ts b/src/carrier-binding.test.ts index 0d8ed5d55..9180f7f33 100644 --- a/src/carrier-binding.test.ts +++ b/src/carrier-binding.test.ts @@ -2419,3 +2419,57 @@ test("revoked expired prepare-only cutover lease cleanup requires exact no-effec assert.deepEqual(replay.lease,result.lease); } finally {f.close();} }); + +function idleCutoverScenario(f:ReturnType,key:string,opts:{revoke?:boolean;advanceMs?:number}={}) { + const context={clientId:"shared-oauth",sessionId:`idle-${key}`}; + const pairing=f.store.requestPairing(context); + const cutover={ + stateRoot:f.root, + attemptKey:key, + currentIdentity:{serverInstanceId:"original",sourceCommit:f.contract.baseRevision,buildId:"old",capabilityManifestSha256:"c".repeat(64)}, + expectedIdentity:{sourceCommit:f.contract.baseRevision,buildId:"old",capabilityManifestSha256:"c".repeat(64)}, + expiresAt:new Date(f.clock()+30_000).toISOString(), + restart:{buildReady:{verifiedBy:"fixture",verifiedAt:new Date(f.clock()).toISOString(),evidence:"prepare-only canary"},actuator:"launchd-self" as const,serviceLabel:"test.service",launchdTarget:"gui/501/test.service"}, + finish:{workspaceId:"ws_idle",agentId:"agt_idle"}, + }; + const contract={...f.contract,scope:[f.root],operations:["cutover_start" as const],cutover,expiresAt:new Date(f.clock()+60_000).toISOString()}; + const approved=f.store.approveLocal(pairing.pendingId,contract); + f.store.redeem(context,pairing.credential); + const plan=planCutoverStart(f.root,cutover); + const lease=f.store.prepareEffect(context,plan.subject); + if(opts.revoke!==false) f.store.revokeLocal(approved.id,approved.version); + f.advance(opts.advanceMs ?? 120_000); + const input={leaseId:lease.leaseId,expectedLeaseVersion:lease.version,carrierId:approved.id,expectedCarrierVersion:opts.revoke===false?approved.version:2,expectedValidityVersion:1,expectedOperationId:plan.operationId,confirmLeaseId:lease.leaseId}; + return {input,lease,plan,approved}; +} + +test("idle orphan lease cleanup fails closed for non-revoked carrier, live lease, pinned lease, durable evidence and wrong identity (#415)", () => { + const leaseRow=(f:ReturnType,id:string)=>f.db.sqlite.prepare("select terminal_state,version from control_plane_resource_leases where lease_id=?").get(id) as {terminal_state:string|null;version:number}; + const expectDenied=(f:ReturnType,s:ReturnType,override:Partial,pattern:RegExp)=>{ + const before=f.snapshot(); + assert.throws(()=>f.store.releaseRevokedIdleCutoverLeaseLocal({...s.input,...override}),pattern); + assert.equal(f.snapshot(),before,"a denied cleanup must not mutate state"); + assert.equal(leaseRow(f,s.lease.leaseId).terminal_state,null); + }; + // Active (not revoked) root carrier. + { const f=fixture();try { const s=idleCutoverScenario(f,"active",{revoke:false}); expectDenied(f,s,{expectedCarrierVersion:2},/exact revoked root carrier/); } finally {f.close();} } + // Revoked carrier but not expired lease/validity. + { const f=fixture();try { const s=idleCutoverScenario(f,"live",{advanceMs:1_000}); expectDenied(f,s,{},/expired/); } finally {f.close();} } + // Wrong confirmation, carrier id, carrier version, lease id. + { const f=fixture();try { const s=idleCutoverScenario(f,"wrong"); + expectDenied(f,s,{confirmLeaseId:"lease_other"},/confirmation must equal the exact lease id/); + expectDenied(f,s,{carrierId:"carrier_nope"},/exact revoked root carrier/); + expectDenied(f,s,{expectedCarrierVersion:s.input.expectedCarrierVersion+1},/exact revoked root carrier/); + expectDenied(f,s,{leaseId:"lease_nope",confirmLeaseId:"lease_nope"},/effect binding changed/); + } finally {f.close();} } + // Pinned lease (active operation handle) must go through reconciliation, never hygiene. + { const f=fixture();try { const s=idleCutoverScenario(f,"pinned"); + f.db.sqlite.prepare("update control_plane_resource_leases set active_operation_handle='pinned-op', operation_state='active' where lease_id=?").run(s.lease.leaseId); + expectDenied(f,s,{},/exact expired unpinned prepare-only lease/); + } finally {f.close();} } + // Terminal witness present. + { const f=fixture();try { const s=idleCutoverScenario(f,"witness"); + f.db.sqlite.prepare("insert into dependency_terminal_witnesses(operation_id,request_hash,lease_id,exit_code,frozen_inputs_unchanged) values(?,?,?,0,1)").run(s.plan.operationId,s.plan.requestHash,s.lease.leaseId); + expectDenied(f,s,{},/refuses any durable or terminal effect evidence/); + } finally {f.close();} } +}); diff --git a/src/control-plane-ownership.test.ts b/src/control-plane-ownership.test.ts index 8679a42f2..9b1ef732f 100644 --- a/src/control-plane-ownership.test.ts +++ b/src/control-plane-ownership.test.ts @@ -483,3 +483,23 @@ test("non-expired missing physical lease still fails closed for unrelated acquis assert.throws(()=>store.acquire(context("two"),request(b,"second")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="INVALID_INPUT"&&/resource identity cannot be resolved/.test(error.message)); } finally {sqlite.close();rmSync(root,{recursive:true,force:true});} }); + +test("expired stale lease with a removed path that overlaps a request blocks with a bounded stale-physical-lease code naming the lease and hygiene command (#415)", () => { + const root=realpathSync.native(mkdtempSync(join(tmpdir(),"devspace-stale-overlap-"))).replaceAll("\\","/"); + const a=join(root,"a"),b=join(root,"b");mkdirSync(a);mkdirSync(b); + const sqlite=db();let now=Date.now(); + const store=new ControlPlaneOwnershipStore(sqlite,{resolveOwnerContext:options.resolveOwnerContext,verifyGrantEvidence:options.verifyGrantEvidence,now:()=>now}); + const request=(path:string,key:string)=>({...input([path]),resourceKind:"filesystem",resourceId:path,resource:path,idempotencyKey:key,expiresAt:new Date(now+60_000).toISOString()}); + try { + const stale=store.acquire(context("one"),{...request(a,"first"),expiresAt:new Date(now+1_000).toISOString()}); + now+=2_000; + rmSync(a,{recursive:true}); + const before=sqlite.prepare("select * from control_plane_resource_leases").all(); + // The parent directory covers the removed path: the stale lease may overlap and must block, not be ignored. + assert.throws(()=>store.acquire(context("two"),request(root,"parent")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="STALE_PHYSICAL_LEASE"&&error.message.includes(stale.leaseId)&&/release-revoked-idle-cutover-lease/.test(error.message)&&error.message.length<400); + assert.deepEqual(sqlite.prepare("select * from control_plane_resource_leases").all(),before); + // A disjoint request is still unaffected, and the stale lease is neither released nor altered. + assert.equal(store.acquire(context("three"),request(b,"disjoint")).resource,b); + assert.equal(store.get(stale.leaseId)?.terminalState,undefined); + } finally {sqlite.close();rmSync(root,{recursive:true,force:true});} +}); diff --git a/src/control-plane-ownership.ts b/src/control-plane-ownership.ts index ff29c4dd6..f6b103cd1 100644 --- a/src/control-plane-ownership.ts +++ b/src/control-plane-ownership.ts @@ -109,7 +109,7 @@ export function resolvePhysicalResource(input: ResourceLeaseInput): Pick Date: Fri, 16 Oct 2026 11:33:42 +0800 Subject: [PATCH 2/3] fix(control-plane): STALE_PHYSICAL_LEASE hint uses the real carrier CLI syntax (#415) Co-Authored-By: Claude Opus 5.5 --- src/control-plane-ownership.test.ts | 2 +- src/control-plane-ownership.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/control-plane-ownership.test.ts b/src/control-plane-ownership.test.ts index 9b1ef732f..7520fde49 100644 --- a/src/control-plane-ownership.test.ts +++ b/src/control-plane-ownership.test.ts @@ -496,7 +496,7 @@ test("expired stale lease with a removed path that overlaps a request blocks wit rmSync(a,{recursive:true}); const before=sqlite.prepare("select * from control_plane_resource_leases").all(); // The parent directory covers the removed path: the stale lease may overlap and must block, not be ignored. - assert.throws(()=>store.acquire(context("two"),request(root,"parent")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="STALE_PHYSICAL_LEASE"&&error.message.includes(stale.leaseId)&&/release-revoked-idle-cutover-lease/.test(error.message)&&error.message.length<400); + assert.throws(()=>store.acquire(context("two"),request(root,"parent")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="STALE_PHYSICAL_LEASE"&&error.message.includes(stale.leaseId)&&/release-revoked-idle-cutover-lease/.test(error.message)&&error.message.length<600); assert.deepEqual(sqlite.prepare("select * from control_plane_resource_leases").all(),before); // A disjoint request is still unaffected, and the stale lease is neither released nor altered. assert.equal(store.acquire(context("three"),request(b,"disjoint")).resource,b); diff --git a/src/control-plane-ownership.ts b/src/control-plane-ownership.ts index f6b103cd1..28d384f4a 100644 --- a/src/control-plane-ownership.ts +++ b/src/control-plane-ownership.ts @@ -324,7 +324,7 @@ export class ControlPlaneOwnershipStore { let unresolved = false; try { this.assertPhysicalBinding(other); } catch (error) { unresolved = error instanceof ControlPlaneOwnershipError && error.code === "INVALID_INPUT" && error.message === "resource identity cannot be resolved"; } - if (unresolved) throw new ControlPlaneOwnershipError("STALE_PHYSICAL_LEASE", `expired lease ${other.leaseId} has an unresolvable physical path and may overlap this scope; resolve it with the Owner-local hygiene command: devspace carrier ${other.leaseId} release-revoked-idle-cutover-lease (only for a revoked, expired, unpinned prepare-only lease with no durable effect)`); + if (unresolved) throw new ControlPlaneOwnershipError("STALE_PHYSICAL_LEASE", `expired lease ${other.leaseId} has an unresolvable path and may overlap this scope; Owner-local hygiene (revoked, expired, unpinned, no durable effect only): devspace carrier release-revoked-idle-cutover-lease ${other.leaseId} --lease-version --carrier --carrier-version --validity-version --operation-id --confirm ${other.leaseId}`); } throw new ControlPlaneOwnershipError("OWNERSHIP_CONFLICT", "overlapping resource scope is already leased"); } From 828b6045df598c21078d64e4d37d331c7da058fc Mon Sep 17 00:00:00 2001 From: James3014 <185240413+James3014@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:50:00 +0800 Subject: [PATCH 3/3] fix(ops): carry cutoverId in the closed-lease Refuse receipt; pin exact recovery command shape in tests (#516) Co-Authored-By: Claude Opus 5.5 --- scripts/ops/m5-cutover/m5-cutover-lib.mjs | 2 +- scripts/ops/m5-cutover/m5-cutover.test.mjs | 2 +- src/control-plane-ownership.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/ops/m5-cutover/m5-cutover-lib.mjs b/scripts/ops/m5-cutover/m5-cutover-lib.mjs index a85f4550c..7d391a61e 100644 --- a/scripts/ops/m5-cutover/m5-cutover-lib.mjs +++ b/scripts/ops/m5-cutover/m5-cutover-lib.mjs @@ -80,7 +80,7 @@ export function selectMode({ file, contract, liveIsOld, liveIsTarget, closedLeas const id = file.cutoverId; throw new Refuse(`cutover ${id} is closed but its state-root lease ${closedLease.leaseId} is not terminal; a new attempt would fail with an overlapping resource scope`, `finish lease replay, do NOT start a new attempt: ${RUNBOOK_CLI} cutover release-terminal-lease --cutover-id ${id} --lease-id ${closedLease.leaseId} --lease-version ${closedLease.version} --carrier --carrier-version --terminal-record-hash --confirm ${id}`, - { activePhase: "closed", leaseId: closedLease.leaseId }); + { activePhase: "closed", cutoverId: id, leaseId: closedLease.leaseId }); } if (!file || file.phase === "closed") return "full"; // Branch a: brand-new attempt. const target = contract.cutover.expectedIdentity; diff --git a/scripts/ops/m5-cutover/m5-cutover.test.mjs b/scripts/ops/m5-cutover/m5-cutover.test.mjs index 8c8d8feb8..33f5590d6 100644 --- a/scripts/ops/m5-cutover/m5-cutover.test.mjs +++ b/scripts/ops/m5-cutover/m5-cutover.test.mjs @@ -59,7 +59,7 @@ test("closed file whose state-root lease is still held is refused with a release const held = { leaseId: "lease_held", version: 3, terminalState: null }; const run = () => selectMode({ file: closed, contract, liveIsOld: false, liveIsTarget: true, closedLease: held }); assert.throws(run, (error) => error instanceof Refuse - && /release-terminal-lease/.test(error.recovery) && /lease_held/.test(error.recovery) && /cut_1/.test(error.recovery) + && /cutover release-terminal-lease --cutover-id cut_1 --lease-id lease_held --lease-version 3 .* --confirm cut_1$/.test(error.recovery) && error.extra.cutoverId === "cut_1" && error.extra.activePhase === "closed" && error.extra.leaseId === "lease_held"); // Liveness only: a terminal lease, an absent lease row, or no lease info keep the brand-new-attempt path. for (const closedLease of [{ leaseId: "lease_held", version: 4, terminalState: "released" }, null, undefined]) { diff --git a/src/control-plane-ownership.test.ts b/src/control-plane-ownership.test.ts index 7520fde49..4348a0269 100644 --- a/src/control-plane-ownership.test.ts +++ b/src/control-plane-ownership.test.ts @@ -496,7 +496,7 @@ test("expired stale lease with a removed path that overlaps a request blocks wit rmSync(a,{recursive:true}); const before=sqlite.prepare("select * from control_plane_resource_leases").all(); // The parent directory covers the removed path: the stale lease may overlap and must block, not be ignored. - assert.throws(()=>store.acquire(context("two"),request(root,"parent")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="STALE_PHYSICAL_LEASE"&&error.message.includes(stale.leaseId)&&/release-revoked-idle-cutover-lease/.test(error.message)&&error.message.length<600); + assert.throws(()=>store.acquire(context("two"),request(root,"parent")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="STALE_PHYSICAL_LEASE"&&error.message.includes(stale.leaseId)&&new RegExp(`carrier release-revoked-idle-cutover-lease ${stale.leaseId} --lease-version .* --confirm ${stale.leaseId}$`).test(error.message)&&error.message.length<600); assert.deepEqual(sqlite.prepare("select * from control_plane_resource_leases").all(),before); // A disjoint request is still unaffected, and the stale lease is neither released nor altered. assert.equal(store.acquire(context("three"),request(b,"disjoint")).resource,b);