diff --git a/scripts/ops/m5-cutover/m5-cutover-lib.mjs b/scripts/ops/m5-cutover/m5-cutover-lib.mjs index 106442482..7d391a61e 100644 --- a/scripts/ops/m5-cutover/m5-cutover-lib.mjs +++ b/scripts/ops/m5-cutover/m5-cutover-lib.mjs @@ -72,7 +72,16 @@ export function recoveryFor(file, liveIsOld, liveIsTarget) { * Decide the sidecar mode from persisted state + live identity (read-only). * Returns "full" | "resume_drain" | "resume_restart" | "resume_finish"; throws Refuse otherwise. */ -export function selectMode({ file, contract, liveIsOld, liveIsTarget }) { +export function selectMode({ file, contract, liveIsOld, liveIsTarget, closedLease }) { + // `closedLease` is the state-root lease row of a CLOSED cutover ({leaseId, version, terminalState}) or null/undefined + // when unknown/absent. The file and the SQLite lease are separate stores: a crash between file close and lease + // finalization leaves a closed file with a still-held lease, and a new attempt would only fail on overlap (#516). + if (file && file.phase === "closed" && closedLease && !closedLease.terminalState) { + const id = file.cutoverId; + throw new Refuse(`cutover ${id} is closed but its state-root lease ${closedLease.leaseId} is not terminal; a new attempt would fail with an overlapping resource scope`, + `finish lease replay, do NOT start a new attempt: ${RUNBOOK_CLI} cutover release-terminal-lease --cutover-id ${id} --lease-id ${closedLease.leaseId} --lease-version ${closedLease.version} --carrier --carrier-version --terminal-record-hash --confirm ${id}`, + { activePhase: "closed", cutoverId: id, leaseId: closedLease.leaseId }); + } if (!file || file.phase === "closed") return "full"; // Branch a: brand-new attempt. const target = contract.cutover.expectedIdentity; const ours = sameDeep(file.oldServerIdentity, contract.cutover.currentIdentity) diff --git a/scripts/ops/m5-cutover/m5-cutover-sidecar.mjs b/scripts/ops/m5-cutover/m5-cutover-sidecar.mjs index 6775d7a77..f27e2d629 100755 --- a/scripts/ops/m5-cutover/m5-cutover-sidecar.mjs +++ b/scripts/ops/m5-cutover/m5-cutover-sidecar.mjs @@ -98,7 +98,16 @@ try { const liveIsTarget = matchesTarget(before, target) && before.mcp.serverInstanceId !== contract.cutover.currentIdentity.serverInstanceId; if (file && file.phase !== "closed") cutoverId = file.cutoverId; - const mode = selectMode({ file, contract, liveIsOld, liveIsTarget }); // "full" | "resume_drain" | "resume_restart" | "resume_finish" + // Closed file but state-root lease not yet terminal (crash between file close and lease finalization): read-only probe. + let closedLease; + if (file?.phase === "closed" && file.coordinationBinding?.leaseId) { + const probeBindings = new CarrierBindingStore(config.stateDir); + try { + const row = probeBindings.ownership.get(file.coordinationBinding.leaseId); + if (row) closedLease = { leaseId: row.leaseId, version: row.version, terminalState: row.terminalState ?? null }; + } finally { probeBindings.close(); } + } + const mode = selectMode({ file, contract, liveIsOld, liveIsTarget, closedLease }); // "full" | "resume_drain" | "resume_restart" | "resume_finish" log("phase_detected", { mode, activePhase: file?.phase ?? null, cutoverId: file?.cutoverId ?? null, liveIsOld, liveIsTarget }); if (mode !== "resume_finish" && Date.parse(contract.cutover.expiresAt) <= Date.now()) { diff --git a/scripts/ops/m5-cutover/m5-cutover.test.mjs b/scripts/ops/m5-cutover/m5-cutover.test.mjs index c88dc4812..33f5590d6 100644 --- a/scripts/ops/m5-cutover/m5-cutover.test.mjs +++ b/scripts/ops/m5-cutover/m5-cutover.test.mjs @@ -54,6 +54,19 @@ test("no active generation or a closed one selects full", () => { assert.equal(select(record({ phase: "closed" }), false, true), "full"); }); +test("closed file whose state-root lease is still held is refused with a release-terminal-lease hint (#516)", () => { + const closed = record({ phase: "closed", coordinationBinding: { leaseId: "lease_held" } }); + const held = { leaseId: "lease_held", version: 3, terminalState: null }; + const run = () => selectMode({ file: closed, contract, liveIsOld: false, liveIsTarget: true, closedLease: held }); + assert.throws(run, (error) => error instanceof Refuse + && /cutover release-terminal-lease --cutover-id cut_1 --lease-id lease_held --lease-version 3 .* --confirm cut_1$/.test(error.recovery) && error.extra.cutoverId === "cut_1" + && error.extra.activePhase === "closed" && error.extra.leaseId === "lease_held"); + // Liveness only: a terminal lease, an absent lease row, or no lease info keep the brand-new-attempt path. + for (const closedLease of [{ leaseId: "lease_held", version: 4, terminalState: "released" }, null, undefined]) { + assert.equal(selectMode({ file: closed, contract, liveIsOld: true, liveIsTarget: false, closedLease }), "full"); + } +}); + test("prepared without progress and old live resumes at drain", () => { assert.equal(select(record(), true, false), "resume_drain"); }); diff --git a/src/carrier-binding.test.ts b/src/carrier-binding.test.ts index 0d8ed5d55..9180f7f33 100644 --- a/src/carrier-binding.test.ts +++ b/src/carrier-binding.test.ts @@ -2419,3 +2419,57 @@ test("revoked expired prepare-only cutover lease cleanup requires exact no-effec assert.deepEqual(replay.lease,result.lease); } finally {f.close();} }); + +function idleCutoverScenario(f:ReturnType,key:string,opts:{revoke?:boolean;advanceMs?:number}={}) { + const context={clientId:"shared-oauth",sessionId:`idle-${key}`}; + const pairing=f.store.requestPairing(context); + const cutover={ + stateRoot:f.root, + attemptKey:key, + currentIdentity:{serverInstanceId:"original",sourceCommit:f.contract.baseRevision,buildId:"old",capabilityManifestSha256:"c".repeat(64)}, + expectedIdentity:{sourceCommit:f.contract.baseRevision,buildId:"old",capabilityManifestSha256:"c".repeat(64)}, + expiresAt:new Date(f.clock()+30_000).toISOString(), + restart:{buildReady:{verifiedBy:"fixture",verifiedAt:new Date(f.clock()).toISOString(),evidence:"prepare-only canary"},actuator:"launchd-self" as const,serviceLabel:"test.service",launchdTarget:"gui/501/test.service"}, + finish:{workspaceId:"ws_idle",agentId:"agt_idle"}, + }; + const contract={...f.contract,scope:[f.root],operations:["cutover_start" as const],cutover,expiresAt:new Date(f.clock()+60_000).toISOString()}; + const approved=f.store.approveLocal(pairing.pendingId,contract); + f.store.redeem(context,pairing.credential); + const plan=planCutoverStart(f.root,cutover); + const lease=f.store.prepareEffect(context,plan.subject); + if(opts.revoke!==false) f.store.revokeLocal(approved.id,approved.version); + f.advance(opts.advanceMs ?? 120_000); + const input={leaseId:lease.leaseId,expectedLeaseVersion:lease.version,carrierId:approved.id,expectedCarrierVersion:opts.revoke===false?approved.version:2,expectedValidityVersion:1,expectedOperationId:plan.operationId,confirmLeaseId:lease.leaseId}; + return {input,lease,plan,approved}; +} + +test("idle orphan lease cleanup fails closed for non-revoked carrier, live lease, pinned lease, durable evidence and wrong identity (#415)", () => { + const leaseRow=(f:ReturnType,id:string)=>f.db.sqlite.prepare("select terminal_state,version from control_plane_resource_leases where lease_id=?").get(id) as {terminal_state:string|null;version:number}; + const expectDenied=(f:ReturnType,s:ReturnType,override:Partial,pattern:RegExp)=>{ + const before=f.snapshot(); + assert.throws(()=>f.store.releaseRevokedIdleCutoverLeaseLocal({...s.input,...override}),pattern); + assert.equal(f.snapshot(),before,"a denied cleanup must not mutate state"); + assert.equal(leaseRow(f,s.lease.leaseId).terminal_state,null); + }; + // Active (not revoked) root carrier. + { const f=fixture();try { const s=idleCutoverScenario(f,"active",{revoke:false}); expectDenied(f,s,{expectedCarrierVersion:2},/exact revoked root carrier/); } finally {f.close();} } + // Revoked carrier but not expired lease/validity. + { const f=fixture();try { const s=idleCutoverScenario(f,"live",{advanceMs:1_000}); expectDenied(f,s,{},/expired/); } finally {f.close();} } + // Wrong confirmation, carrier id, carrier version, lease id. + { const f=fixture();try { const s=idleCutoverScenario(f,"wrong"); + expectDenied(f,s,{confirmLeaseId:"lease_other"},/confirmation must equal the exact lease id/); + expectDenied(f,s,{carrierId:"carrier_nope"},/exact revoked root carrier/); + expectDenied(f,s,{expectedCarrierVersion:s.input.expectedCarrierVersion+1},/exact revoked root carrier/); + expectDenied(f,s,{leaseId:"lease_nope",confirmLeaseId:"lease_nope"},/effect binding changed/); + } finally {f.close();} } + // Pinned lease (active operation handle) must go through reconciliation, never hygiene. + { const f=fixture();try { const s=idleCutoverScenario(f,"pinned"); + f.db.sqlite.prepare("update control_plane_resource_leases set active_operation_handle='pinned-op', operation_state='active' where lease_id=?").run(s.lease.leaseId); + expectDenied(f,s,{},/exact expired unpinned prepare-only lease/); + } finally {f.close();} } + // Terminal witness present. + { const f=fixture();try { const s=idleCutoverScenario(f,"witness"); + f.db.sqlite.prepare("insert into dependency_terminal_witnesses(operation_id,request_hash,lease_id,exit_code,frozen_inputs_unchanged) values(?,?,?,0,1)").run(s.plan.operationId,s.plan.requestHash,s.lease.leaseId); + expectDenied(f,s,{},/refuses any durable or terminal effect evidence/); + } finally {f.close();} } +}); diff --git a/src/control-plane-ownership.test.ts b/src/control-plane-ownership.test.ts index 8679a42f2..4348a0269 100644 --- a/src/control-plane-ownership.test.ts +++ b/src/control-plane-ownership.test.ts @@ -483,3 +483,23 @@ test("non-expired missing physical lease still fails closed for unrelated acquis assert.throws(()=>store.acquire(context("two"),request(b,"second")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="INVALID_INPUT"&&/resource identity cannot be resolved/.test(error.message)); } finally {sqlite.close();rmSync(root,{recursive:true,force:true});} }); + +test("expired stale lease with a removed path that overlaps a request blocks with a bounded stale-physical-lease code naming the lease and hygiene command (#415)", () => { + const root=realpathSync.native(mkdtempSync(join(tmpdir(),"devspace-stale-overlap-"))).replaceAll("\\","/"); + const a=join(root,"a"),b=join(root,"b");mkdirSync(a);mkdirSync(b); + const sqlite=db();let now=Date.now(); + const store=new ControlPlaneOwnershipStore(sqlite,{resolveOwnerContext:options.resolveOwnerContext,verifyGrantEvidence:options.verifyGrantEvidence,now:()=>now}); + const request=(path:string,key:string)=>({...input([path]),resourceKind:"filesystem",resourceId:path,resource:path,idempotencyKey:key,expiresAt:new Date(now+60_000).toISOString()}); + try { + const stale=store.acquire(context("one"),{...request(a,"first"),expiresAt:new Date(now+1_000).toISOString()}); + now+=2_000; + rmSync(a,{recursive:true}); + const before=sqlite.prepare("select * from control_plane_resource_leases").all(); + // The parent directory covers the removed path: the stale lease may overlap and must block, not be ignored. + assert.throws(()=>store.acquire(context("two"),request(root,"parent")),(error:unknown)=>error instanceof ControlPlaneOwnershipError&&error.code==="STALE_PHYSICAL_LEASE"&&error.message.includes(stale.leaseId)&&new RegExp(`carrier release-revoked-idle-cutover-lease ${stale.leaseId} --lease-version .* --confirm ${stale.leaseId}$`).test(error.message)&&error.message.length<600); + assert.deepEqual(sqlite.prepare("select * from control_plane_resource_leases").all(),before); + // A disjoint request is still unaffected, and the stale lease is neither released nor altered. + assert.equal(store.acquire(context("three"),request(b,"disjoint")).resource,b); + assert.equal(store.get(stale.leaseId)?.terminalState,undefined); + } finally {sqlite.close();rmSync(root,{recursive:true,force:true});} +}); diff --git a/src/control-plane-ownership.ts b/src/control-plane-ownership.ts index ff29c4dd6..28d384f4a 100644 --- a/src/control-plane-ownership.ts +++ b/src/control-plane-ownership.ts @@ -109,7 +109,7 @@ export function resolvePhysicalResource(input: ResourceLeaseInput): Pick --carrier --carrier-version --validity-version --operation-id --confirm ${other.leaseId}`); + } throw new ControlPlaneOwnershipError("OWNERSHIP_CONFLICT", "overlapping resource scope is already leased"); } try {