From fe47d298c227268613f6b9b480c9aefbde7c1b32 Mon Sep 17 00:00:00 2001 From: Paul Fidika Date: Tue, 11 Aug 2026 13:44:23 -0600 Subject: [PATCH 1/4] =?UTF-8?q?pgw#1141:=20a=20boot-adopted=20cell=20SERVE?= =?UTF-8?q?S=20=E2=80=94=20the=20warmup=20proof=20barrier=20stops=20destro?= =?UTF-8?q?ying=20evidence=20stronger=20than=20the=20evidence=20it=20deman?= =?UTF-8?q?ds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real pods (RTX 4000 Ada, gen-worker 0.106.0, hub 7ae35d54a2) did this, identically: `boot_adopt=hit` -> materialize -> `cell_numerics cos=1.00000 ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored that same artifact `unexercised`, folded it into `unproven`, wrote `function_proofs[id]=set()` and called `aot_serve.unwrap` / `compile_cache.unwrap`. `functions=()` then made `_install_compile_targets` omit the target (`target_applicability_incomplete`), which `_assert_armed_targets_installed` correctly reported as `armed_target_unresolved` — pgw#1093's own token on its boot-adopt twin — and the pod served eager for life and published nothing. The SELF-MINT arm was healthy on the same wheel, card and release, and the asymmetry is structural: a mint's warmup DRIVES its own capture, so it dispatches; an adopt arms BEFORE setup, so by construction nothing has. Paul's ruling, verbatim: "skip the warmup / arm check, so we can serve right away … try to serve, if an error is encountered, and it's the cause of the cell, de-arm the cell, and serve eager instead. If our cell is correct this adds zero cost. If our cell is wrong we detect immediately and then fall back to working." So this is mostly a deletion. On the EXPORTED lane the warm ledger gates nothing: the arm stands, and `_install_compile_targets` permits the whole contract the cell advertises instead of whatever the warm plan happened to dispatch. An AOTI artifact is ahead-of-time machine code for this exact sm/toolchain — the first call is full speed, so the warm pass never made it faster, it only checked it. What keeps the teeth: * the pgw#868 numerics gate still REFUSES a cell that does not reproduce eager — the only detector for a cell that runs cleanly and returns a WRONG image, which try-serve cannot see; * a cell-attributable failure at serve time answers THAT request eager, revokes the arm stickily, and says so typed (the existing wrapper); * an out-of-envelope shape is refused by name, per request (pgw#844); * evidence AGAINST still disarms; only the ABSENCE of evidence stopped being a verdict; * publishing to the fleet stays evidence-gated — serving optimistically costs this pod one eager fallback, publishing an unverified cell costs every pod that adopts it. The DYNAMO lane keeps its ledger, and the reason is the failure MODE, not the vintage: a dynamo arm that does not serve its cell RECOMPILES — correct output, silently slower, no exception for try-serve to catch, and no numerics gate on that lane at all. Its per-class cache-hit proof is the only detector that exists, so deleting it would remove a detector with no replacement. Boot cost deleted with the barrier: `tracing` now means a capture is being DRIVEN (or a dynamo arm is live), not "something is armed", so a boot adopting an exported cell runs the collapsed EAGER warm plan instead of the full class cross-product. sdxl: 18 full generates per handler (9 aspect buckets x 2 guidance classes) -> 2. The pgw#654 / ie#546 canary priced this same machinery at ~30-minute first boots. Serve-time attribution is honest now: a CUDA OOM is a fact about the CARD at that instant (a sibling load, a rotation), so it serves the request eager and leaves the cell ARMED — condemning it would retire a correct artifact on the first busy moment and re-mint it on the replacement pod. Accepted tail risk, recorded rather than hidden: an artifact pathological enough to corrupt the CUDA context is not recoverable in-request (process restart). With key-matched, numerics-gated cells that residual is judged acceptable. RED both directions, nothing stubbed: `tests/test_adopted_cell_warm_proof_ pgw1141.py` drives the real `provision.arm_aot` (real packed artifact, real gate, real ladder), the real `ensure_setup` warmup proof, and the real serving wrapper. Six rows fail on origin/master — including one whose captured log is the pod's own line verbatim: `_Pipe armed=False targets_resolve=True`. --- changelog.d/pgw1141.md | 1 + src/gen_worker/aot_serve.py | 86 ++++ src/gen_worker/executor.py | 229 ++++++++- src/gen_worker/models/provision.py | 18 +- src/gen_worker/numerics_ladder.py | 14 +- tests/test_adopted_cell_warm_proof_pgw1141.py | 483 ++++++++++++++++++ tests/test_aot_boot_proof_gap_pgw735.py | 39 +- 7 files changed, 834 insertions(+), 36 deletions(-) create mode 100644 changelog.d/pgw1141.md create mode 100644 tests/test_adopted_cell_warm_proof_pgw1141.py diff --git a/changelog.d/pgw1141.md b/changelog.d/pgw1141.md new file mode 100644 index 000000000..30cc127af --- /dev/null +++ b/changelog.d/pgw1141.md @@ -0,0 +1 @@ +- **pgw#1141: a boot-adopted cell SERVES — the setup warmup no longer disarms an exported artifact for want of a dispatch it could not have had.** Measured identically on two real pods (RTX 4000 Ada, 0.106.0): `boot_adopt=hit` -> materialize -> `cell_numerics cos=1.00000 ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored that same artifact `unexercised`, folded it into `unproven`, set `function_proofs[id]=set()` and called `aot_serve.unwrap` / `compile_cache.unwrap`. The pod served eager for life and published nothing, while the SELF-MINT arm on the same wheel, card and release served `+compiled` — because a mint's warmup drives its own capture and therefore dispatches, and an adopt arms BEFORE setup and therefore cannot. The gate was destroying evidence stronger than the evidence it demanded. Paul's ruling (2026-08-11): *"skip the warmup/arm check, so we can serve right away… try to serve, and if an error is encountered and it's the cause of the cell, de-arm the cell and serve eager instead. If our cell is correct this adds zero cost."* An AOTI artifact is ahead-of-time machine code for this exact sm/toolchain — the first call is full speed, and the warm pass never made it faster, it only checked it. On the exported lane the warm ledger now gates NOTHING: the arm stands, `_install_compile_targets` permits the whole advertised contract instead of `function_proofs` (which produced `functions=()` -> `target_applicability_incomplete` -> `armed_target_unresolved`, pgw#1093's token on its boot-adopt twin), and correctness is carried by the three detectors that can actually see a fault — the pgw#868 numerics gate refuses a cell that does not reproduce eager (the only detector for a cell that runs cleanly and returns a WRONG image), a cell-attributable failure at serve time answers that request eager and revokes the arm stickily and typed, and an out-of-envelope shape is refused by name per request. Publishing to the fleet stays evidence-gated: an artifact with neither a warm dispatch nor a standing parity verdict serves, but withholds its cell. **The DYNAMO lane keeps its ledger** — a dynamo arm that does not serve its cell RECOMPILES (correct output, silently slower, no exception to catch, and no numerics gate on that lane), so its per-class cache-hit proof is the only detector in existence. **Boot cost deleted with it:** `tracing` now means a capture is being driven or a dynamo arm is live, so a boot adopting an exported cell runs the collapsed EAGER warm plan instead of the full class cross-product — for sdxl that is 18 full generates per handler (9 aspect buckets x 2 guidance classes) reduced to 2; the pgw#654/ie#546 canary priced the same machinery at ~30-minute first boots. The parity verdict is banked on the arm that earned it (`aot_serve.NumericsProof`, scoped to the marker so `unwrap` drops it and a revoked target withdraws it), and serve-time attribution is honest: a CUDA OOM is a fact about the card at that instant, so it serves the request eager and leaves the cell armed rather than condemning a correct artifact on the first busy moment. Accepted tail risk, written down rather than hidden: an artifact pathological enough to corrupt the CUDA context is not recoverable in-request. `tests/test_adopted_cell_warm_proof_pgw1141.py` drives the real `provision.arm_aot` (real packed artifact, real gate, real ladder), the real `ensure_setup` warmup proof, and the real serving wrapper — 6 rows red before the fix, both directions plus the OOM attribution. diff --git a/src/gen_worker/aot_serve.py b/src/gen_worker/aot_serve.py index 4e586e808..2ea737a7b 100644 --- a/src/gen_worker/aot_serve.py +++ b/src/gen_worker/aot_serve.py @@ -105,6 +105,7 @@ sku_slug, ) from .models import lora_lifted +from .models.memory import is_cuda_oom logger = logging.getLogger(__name__) @@ -2512,6 +2513,26 @@ def aot_forward(*args: Any, **kwargs: Any) -> Any: _revoke(state, f"constants unbound: {exc}") return original(*args, **eager_kwargs) except Exception as exc: # noqa: BLE001 — ANY artifact problem => eager + if is_cuda_oom(exc): + # pgw#1141: ATTRIBUTION. The serve-first doctrine makes the + # first real request the proof, so what that request blames + # decides whether a good cell survives — and allocator + # exhaustion is a fact about the CARD at this instant (a + # sibling load, a concurrent rotation), not about the artifact. + # Condemning the cell for it would retire a correct one on the + # first busy moment and re-mint it on the replacement pod. + # Serve THIS request eager, stay armed, say so. + logger.warning( + "aot-serve: %s hit CUDA OOM (%s); serving this request " + "eager, artifact stays armed — allocator pressure is not " + "the cell's fault", label, exc) + activity_mod.emit_event( + "aot_serve_oom", + f"family={meta.get('family')} target={label}: " + f"{type(exc).__name__}: {exc}", + phase="cuda_oom", + ) + return original(*args, **eager_kwargs) state["failed"] = True detail = ( f"AOTI artifact {label} failed: " @@ -2937,6 +2958,67 @@ def served_entry_calls(pipeline: Any) -> Dict[str, int]: return out +@dataclass(frozen=True) +class NumericsProof: + """What the pgw#868 gate MEASURED on THIS arm, on THIS pod (pgw#1141). + + The gate already runs the artifact — every packaged entry, through its own + runner, on the entry's own declared feed — against the eager forward it + replaces. That is an execution proof and an accuracy proof in one, and it + was discarded the moment it was announced, so the setup warmup then scored + the same artifact ``unexercised`` (no dispatch during ITS window) and + destroyed it: two real pods resolved, materialized and verified a cell at + ``cos=1.00000`` on 3/3 axes and served eager for life. + + Kept on the arm's own marker, so it is scoped to exactly the wrap that + earned it: :func:`unwrap` drops the marker and a re-arm replaces it, which + means a proof can never outlive the artifact it is about. + """ + + cell_key: str + axes: int + worst_entry: str + worst_cosine: float + verdict: str + elapsed_ms: int = 0 + + def __str__(self) -> str: + return (f"cell_numerics axes={self.axes} worst={self.worst_entry} " + f"cos={self.worst_cosine:.5f} {self.verdict} " + f"key={self.cell_key or '?'} took={self.elapsed_ms}ms") + + +def record_numerics_proof(pipeline: Any, proof: NumericsProof) -> bool: + """Bank the parity verdict on the live arm. False when nothing is armed.""" + marker = getattr(pipeline, _MARKER_ATTR, None) + if not isinstance(marker, dict) or not is_armed(pipeline): + return False + marker["numerics"] = proof + return True + + +def numerics_measured(pipeline: Any) -> bool: + """A parity verdict was banked on this marker, whether or not the cell is + still armed — the discriminator between "nobody measured it" and "it was + measured and then revoked", which are different defects.""" + marker = getattr(pipeline, _MARKER_ATTR, None) or {} + return isinstance(marker.get("numerics"), NumericsProof) + + +def numerics_proof(pipeline: Any) -> Optional[NumericsProof]: + """The parity proof this pod took on the CURRENTLY armed artifact. + + ``None`` unless a proof was banked AND the cell is still armed — the same + two-halves rule as :func:`proven_since`, for the same reason: an artifact + that ran and then revoked has not proven anything. + """ + marker = getattr(pipeline, _MARKER_ATTR, None) or {} + proof = marker.get("numerics") + if not isinstance(proof, NumericsProof) or not is_armed(pipeline): + return None + return proof + + def proven_since(pipeline: Any, before: int) -> bool: """The exported lane's ADOPTION PROOF (pgw#735). @@ -3079,6 +3161,10 @@ def unwrap(pipeline: Any) -> bool: "load_and_wrap", "marshal_positional", "note_aot_key", + "NumericsProof", + "numerics_measured", + "numerics_proof", + "record_numerics_proof", "pack", "range_digest", "resident_constants", diff --git a/src/gen_worker/executor.py b/src/gen_worker/executor.py index 2050afa06..d7404c364 100644 --- a/src/gen_worker/executor.py +++ b/src/gen_worker/executor.py @@ -199,7 +199,7 @@ from .models.execution_lane_gate import ExecutionLaneGate, arm_execution_lane_gate from .models.memory import rearm_offload from . import fleet_cells -from . import aot_serve, shape_growth, trt_engine +from . import aot_serve, numerics_ladder, shape_growth, trt_engine from . import fleet_cells as fleet_cells_mod from . import hot_swap from . import mint_delegate @@ -4689,13 +4689,39 @@ def _install_compile_targets( for slot, ref, digest in bindings ) and len({slot for slot, _ref, _digest in bindings}) == len(bindings) active_selection = active_artifacts.get(id(pipeline)) + # pgw#1141 (Paul's ruling, 2026-08-11): on the EXPORTED lane the + # warm ledger GATES NOTHING. It used to decide which aliases an + # installed target may serve, so an object the boot warmup happened + # not to dispatch through was handed `permitted_names=set()` -> + # `function_names=()` -> `target_applicability_incomplete` -> a pod + # that had just verified its cell at `cos=1.00000` served eager for + # life. An AOTI artifact is ahead-of-time machine code for this + # exact sm/toolchain: the first call is full speed, and the warm + # pass never made it faster — it only checked it. What the cell + # advertises is what it may serve; a class it does not carry is + # refused BY NAME at ingress and served eager per request + # (pgw#844), and a cell-attributable failure revokes the arm + # in-request through the wrapper's own fallback. + # + # The DYNAMO lane keeps the ledger, and the difference is the + # failure MODE, not the vintage: a dynamo arm that does not serve + # its cell RECOMPILES — correct output, silently slower, no + # exception for try-serve to catch and no numerics gate on that + # lane at all. Its per-class cache-hit ledger is the only detector + # that exists, so deleting it would remove a detector with no + # replacement. + exported_arm = bool( + active_selection is not None + and aot_serve.is_aot_ref(active_selection.ref)) permitted_names = ( - function_proofs[id(pipeline)] + contract_names if exported_arm + else function_proofs[id(pipeline)] if id(pipeline) in function_proofs else contract_names ) object_proven_by_custom_warmup = bool( - spec.cls is not None + not exported_arm + and spec.cls is not None and callable(getattr(spec.cls, "warmup", None)) and function_proofs.get(id(pipeline)) ) @@ -6317,7 +6343,7 @@ async def _run_synthesized_warmup( snapshots: Optional[Dict[str, pb.Snapshot]], *, proof_objects: typing.Iterable[Any] = (), - cold_proof_ids: typing.Container[int] = (), + cold_proof_ids: typing.Collection[int] = (), allow_contract_skip: bool = False, armed_cell_refs: typing.Iterable[str] = (), ) -> _WarmupEvidence: @@ -6356,13 +6382,27 @@ async def _run_synthesized_warmup( logger.info("boot warmup skipped for %s: %s", skip.spec.name, skip.reason) objects = tuple({id(obj): obj for obj in proof_objects}.values()) - # Tracing == some artifact is armed or minting on this setup; only - # then does the full class x bucket cross-product buy anything (each - # graph must trace into the capture / prove against the cell). - tracing = bool(objects) memory = self._warm_contract_runs.setdefault( self._warm_contract_key(spec), set()) armed_refs = tuple(armed_cell_refs) + # pgw#1141: TRACING is what makes the full class x bucket cross-product + # worth its wall time — a capture being DRIVEN (every declared graph + # must trace into it), or a DYNAMO arm, whose only detector of a + # non-serving cell is the per-class cache-hit ledger this plan feeds. + # + # It used to mean "some artifact is armed", which charged a boot + # ADOPTING AN EXPORTED CELL the whole cross-product (sdxl: 9 aspect + # buckets x 2 guidance classes = 18 full generates per handler) for a + # proof that no longer gates anything on that lane. An AOTI cell is + # ahead-of-time machine code for this exact sm/toolchain: the first + # call is full speed, so re-running every declared class before READY + # buys nothing the numerics gate and the in-request fallback do not + # already buy — it is pure boot latency the tenant waits through. Such + # a boot now runs the EAGER plan (one collapsed run per (function, + # guidance class)), which is the allocator/kernel warm the eager + # fallback path needs anyway. + tracing = bool(cold_proof_ids) or any( + not aot_serve.is_aot_ref(ref) for ref in armed_refs) skip_ok = ( allow_contract_skip and not cold_proof_ids @@ -7442,6 +7482,12 @@ async def run_warmup() -> Tuple[int, Dict[int, set[str]], str]: # adoption. Zero proven objects still fails closed (gw#586). disproven: list[_CompileObjectCandidate] = [] unexercised: list[_CompileObjectCandidate] = [] + #: pgw#1141: per object, why the boot warmup landed no dispatch + #: on an ARMED artifact — the posture that follows is a row on + #: the wire, not something a reader has to infer from the two + #: later events that only describe its consequences + #: (`target_applicability_incomplete`, `armed_target_unresolved`). + arm_without_dispatch: Dict[int, str] = {} proven = 0 hits = 0 misses = 0 @@ -7467,8 +7513,57 @@ async def run_warmup() -> Tuple[int, Dict[int, set[str]], str]: proved_sel = inj.active_compile_artifacts.get(id(pipe)) if proved_sel is not None: compile_cache.record_cell_proven(proved_sel.ref) - else: + continue + # pgw#1141: THE second proof, and on a boot-ADOPTED + # cell it is the only one that can exist. The arm + # happens before setup, so nothing has dispatched yet + # by construction — but the pgw#868 gate has already + # run EVERY packaged entry through its own runner + # against the eager forward it replaces, on this pod, + # on these weights, on this card. That is a superset of + # what `proven_since` asks (it executed, and it is + # still armed) plus the accuracy the dispatch counter + # cannot see. Scoring it `unexercised` destroyed + # artifacts verified at cos=1.00000 on two real pods, + # and left the SELF-MINT arm — which reaches its proof + # by a warm dispatch — as the only way to serve + # compiled. Both arms take the same measurement in + # `provision.arm_aot`, so reading it here is what makes + # adopted and self-minted cells symmetrical rather + # than a special case for one of them. + parity = aot_serve.numerics_proof(pipe) + if parity is None: + arm_without_dispatch[id(pipe)] = ( + "artifact revoked since it was measured" + if aot_serve.numerics_measured(pipe) + else "no parity verdict was banked on this arm") unexercised.append(candidate) + continue + proven += 1 + # `function_proofs` is deliberately NOT narrowed to + # `spec.name` here: the parity verdict covers every + # entry the cell PACKAGES, i.e. the whole contract its + # key advertises, so the install falls through to + # `contract_names`. A class the cell does not carry is + # refused BY NAME at ingress and served eager per + # request (pgw#844), which is the backstop that lets + # this attribution be contract-wide. + proved_sel = inj.active_compile_artifacts.get(id(pipe)) + if proved_sel is not None: + compile_cache.record_cell_proven(proved_sel.ref) + logger.info( + "compile object (slots=%s) proven by its own " + "numerics parity (%s); the boot warmup landed no " + "dispatch on it (pgw#1141)", + sorted(candidate.slots), parity) + activity_mod.emit_event( + activity_mod.KIND_CELL_NUMERICS, + f"{spec.name}: the boot warmup dispatched no call " + f"through this armed cell, and its own parity " + f"measurement on THIS pod carries the proof " + f"instead — {parity}", + phase=numerics_ladder.PHASE_SERVING_PROOF, + ) continue before = proof_before.get(id(pipe)) if before is None: @@ -7481,6 +7576,9 @@ async def run_warmup() -> Tuple[int, Dict[int, set[str]], str]: hits += max(0, pipe_hits) misses += max(0, pipe_misses) if not warmed or calls <= 0: + arm_without_dispatch[id(pipe)] = ( + "the dynamo lane takes no parity measurement, so " + "this boot holds no evidence either way") unexercised.append(candidate) elif pipe_hits > 0: proven += 1 @@ -7526,10 +7624,85 @@ async def run_warmup() -> Tuple[int, Dict[int, set[str]], str]: # the bookkeeping down to readiness — reports an honest # phase instead of a stale seal_publish. activity_mod.current_phase(activity_mod.PHASE_FINALIZE) + + def _confess_arm_without_dispatch( + candidate: "_CompileObjectCandidate", + ) -> None: + """pgw#1141: name the DECISION, at the decision point. + + Every emission the old disarm produced described its + wreckage two frames later (`target_applicability_ + incomplete`, then `armed_target_unresolved`), so a reader + had to infer that an armed, resolvable cell had been thrown + away. The decision is the opposite one now — the arm STANDS + — and it is still a row, because an unannounced posture is + indistinguishable from a gate that never ran.""" + reason = arm_without_dispatch.get(id(candidate.pipeline), "") + if not reason: + return + activity_mod.emit_event( + activity_mod.KIND_CELL_NUMERICS, + f"{spec.name}: the exported cell on slots " + f"{sorted(candidate.slots)} took no warm dispatch and " + f"{reason} — it STAYS ARMED and serves; a " + f"cell-attributable failure revokes it in-request, and " + f"it publishes nothing to the fleet", + phase=numerics_ladder.PHASE_PROOF_ABSENT, + ) + + # pgw#1141 (Paul's ruling, 2026-08-11), and it is a DELETION: + # *"skip the warmup/arm check, so we can serve right away; try + # to serve, and if an error is encountered and it is the cell's + # fault, de-arm the cell and serve eager instead. If our cell is + # correct this adds zero cost."* An ABSENCE of warm evidence is + # no longer a verdict about the artifact — an adopted cell arms + # before setup, so nothing has dispatched through it BY + # CONSTRUCTION, and disarming on that destroyed cells verified + # at cos=1.00000 on two real pods while the self-mint arm (which + # gets its dispatch from the warmup that drives its own capture) + # sailed through. The two arms are symmetrical now: neither is + # disarmed for want of a dispatch. + # + # SCOPED TO THE EXPORTED LANE, because the difference is the + # failure MODE. An AOTI cell that cannot serve RAISES, and the + # wrapper answers that request eager; a DYNAMO arm that does not + # serve its cell RECOMPILES — correct output, silently slower, + # no exception for try-serve to catch and no numerics gate on + # that lane at all — so its per-class cache-hit ledger is the + # only detector in existence and keeps its teeth. + # + # What still has teeth, unchanged: + # * the pgw#868 numerics gate REFUSES a cell that does not + # reproduce eager — the only detector for a cell that runs + # cleanly and returns a WRONG image, which try-serve cannot + # see; + # * EVIDENCE AGAINST still disarms (`disproven`: the object was + # exercised and demonstrably did not serve its own graph — + # a measured fault, not a missing measurement); + # * a cell-attributable failure at serve time revokes the arm + # IN-REQUEST (`aot_serve.wrap_module` / the pgw#680 + # guard-miss doctrine): the tenant still gets a correct eager + # answer, the disarm is sticky for the process, and it is + # typed on the wire. + # * PUBLISHING to the fleet stays evidence-gated below — + # serving optimistically costs this pod one eager fallback, + # publishing an unverified cell costs every pod that adopts + # it. unproven = list(disproven) - if not proven: - unproven.extend(unexercised) - unexercised = [] + # The DYNAMO lane's silent-recompile detector, unchanged: with + # nothing proven, an unexercised dynamo object is still folded + # in and disarmed. Exported candidates never reach here — they + # are scored above and keep their arm either way. + dynamo_unexercised = [ + candidate for candidate in unexercised + if id(candidate.pipeline) not in aot_proof_before + ] + if not proven and dynamo_unexercised: + unproven.extend(dynamo_unexercised) + unexercised = [ + candidate for candidate in unexercised + if candidate not in dynamo_unexercised + ] if unproven: quant_execution_lane = any( @@ -7540,6 +7713,7 @@ async def run_warmup() -> Tuple[int, Dict[int, set[str]], str]: for candidate in unproven: pipe = candidate.pipeline function_proofs[id(pipe)] = set() + _confess_arm_without_dispatch(candidate) # pgw#722 finding 2: an exported arm disarms through # its own lane — aot_serve.unwrap restores the # forward it captured (under the F2 flip that is the @@ -7649,6 +7823,27 @@ async def run_warmup() -> Tuple[int, Dict[int, set[str]], str]: logger.warning("%s; serving eager", detail) for candidate in unexercised: pipe = candidate.pipeline + if id(pipe) in aot_proof_before: + # THE DELETED BARRIER (pgw#1141). This branch used to + # unwrap the artifact, drop the lifted lanes, pop the + # active selection and abandon the mint — for an object + # the warm plan simply never dispatched through, which + # is EVERY boot-adopted cell by construction. It keeps + # the arm now and says so; the publish half is the only + # decision an absent measurement may still make. + logger.warning( + "compile object (slots=%s) armed with no warm " + "dispatch (calls=0); it SERVES, and a " + "cell-attributable failure revokes it in-request " + "(pgw#1141)", sorted(candidate.slots)) + _confess_arm_without_dispatch(candidate) + if aot_serve.numerics_proof(pipe) is None: + # Nothing on this pod has evidence this artifact + # works: it may serve (try-serve decides), but it + # must not be PUBLISHED to pods that cannot + # re-derive that verdict. + self._abandon_pending_mint(inj, pipe) + continue mandatory = pipeline_weight_lane(pipe).startswith( _MANDATORY_EXECUTION_LANES) if mandatory: @@ -7662,17 +7857,15 @@ async def run_warmup() -> Tuple[int, Dict[int, set[str]], str]: "proof covers only its exercised siblings", sorted(candidate.slots)) continue + # The dynamo lane keeps its disarm: an unexercised dynamo + # object that starts serving RECOMPILES silently, and no + # detector downstream would ever say so. logger.warning( "compile object (slots=%s) unproven (no warmup " "modality, calls=0); serving eager", sorted(candidate.slots)) + _confess_arm_without_dispatch(candidate) function_proofs[id(pipe)] = set() - # pgw#722 finding 2: same exported-lane disarm as the - # unproven loop above. - if aot_serve.unwrap(pipe): - from .models import lora_lifted - - lora_lifted.remove_lifted_lora_execution_lanes(pipe) compile_cache.unwrap(pipe) if spec.lora_bucket: compile_cache.drop_lora_execution_lane(pipe) diff --git a/src/gen_worker/models/provision.py b/src/gen_worker/models/provision.py index 4942edc8a..c42a37a15 100644 --- a/src/gen_worker/models/provision.py +++ b/src/gen_worker/models/provision.py @@ -513,13 +513,27 @@ def gate_cell_numerics(pipe: Any, cfg: Any) -> bool: else "no_axis_measured"), f"{report.context()} | unmeasured: {rows}") comparison = report.comparison() + worst = report.worst() try: numerics_ladder.gate( comparison, kind=activity_mod.KIND_CELL_NUMERICS, - refuse=lambda detail, worst: numerics_probe.CellNumericsRefused( - detail, worst), + refuse=lambda detail, worst_row: numerics_probe.CellNumericsRefused( + detail, worst_row), context=report.context()) + # pgw#1141: the verdict is BANKED on the arm, not merely announced. + # This measurement executed every packaged entry through its own + # runner and matched eager — so the setup warmup, which scores an + # exported arm by "did a dispatch land during MY window", has a proof + # to read instead of destroying the artifact that carries it. + aot_serve.record_numerics_proof(pipe, aot_serve.NumericsProof( + cell_key=str(report.cell_key or ""), + axes=len(report.verdicts), + worst_entry=(worst.axis.name if worst is not None else ""), + worst_cosine=(comparison.cosine if comparison is not None else 0.0), + verdict=(comparison.verdict if comparison is not None else ""), + elapsed_ms=report.elapsed_ms, + )) if comparison is not None and comparison.healthy: activity_mod.emit_event( activity_mod.KIND_CELL_NUMERICS, diff --git a/src/gen_worker/numerics_ladder.py b/src/gen_worker/numerics_ladder.py index 7768a6dae..84f849326 100644 --- a/src/gen_worker/numerics_ladder.py +++ b/src/gen_worker/numerics_ladder.py @@ -53,6 +53,17 @@ PHASE_DEGRADED = "degraded" PHASE_REFUSED = "refused" +#: pgw#1141 — the two phases the BOOT WARMUP proof gate reports against a +#: parity verdict, on this same kind so one query (`?kind=cell_numerics`) +#: answers what happened to every cell that armed on a pod. +#: ``serving_proof``: the boot warmup landed no dispatch on an armed exported +#: cell and the pod's OWN parity measurement carried the proof instead (an +#: unannounced pass is indistinguishable from a gate that never ran). +#: ``proof_absent``: no dispatch AND no standing parity verdict — the artifact +#: is disarmed, and this row says which half was missing. +PHASE_SERVING_PROOF = "serving_proof" +PHASE_PROOF_ABSENT = "proof_absent" + def worst_of(*verdicts: str) -> str: """The worst rung among ``verdicts`` (unknown values read as HEALTHY).""" @@ -438,7 +449,8 @@ def declared_thresholds(cfg: Any) -> Thresholds: __all__ = [ "DEFAULT_THRESHOLDS", "NUMERICS_FLOOR", "NUMERICS_RETENTION_FLOOR", "NUMERICS_WARN", - "PHASE_DEGRADED", "PHASE_REFUSED", + "PHASE_DEGRADED", "PHASE_PROOF_ABSENT", "PHASE_REFUSED", + "PHASE_SERVING_PROOF", "VERDICT_DEGRADED", "VERDICT_DESTROYED", "VERDICT_HEALTHY", "Comparison", "RowStat", "Thresholds", "compare_outputs", "declared_thresholds", "flatten_outputs", "gate", diff --git a/tests/test_adopted_cell_warm_proof_pgw1141.py b/tests/test_adopted_cell_warm_proof_pgw1141.py new file mode 100644 index 000000000..b73840ef1 --- /dev/null +++ b/tests/test_adopted_cell_warm_proof_pgw1141.py @@ -0,0 +1,483 @@ +"""pgw#1141: a boot-ADOPTED cell carries its own proof, and the setup warmup +proof gate must READ it instead of destroying the artifact that holds it. + +MEASURED, twice, on two real pods (RTX 4000 Ada, gen-worker 0.106.0, hub +`7ae35d54a2`): `boot_adopt=hit` -> materialize -> `cell_numerics cos=1.00000 +ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored the +same artifact `unexercised` (it dispatched nothing through it — the adopt arms +BEFORE setup, so by construction nothing has), folded it into `unproven`, +`function_proofs[id]=set()`, `aot_serve.unwrap`, `compile_cache.unwrap`. The +pod served eager for life and published nothing. The self-mint arm on the SAME +wheel, card and release served `+compiled` with an empty `fallback_reason`, +because it reaches its proof through a warm dispatch. + +So the gate was destroying evidence STRONGER than the evidence it demands. The +pgw#868 numerics gate runs every packaged entry through its own runner against +the eager forward it replaces — that is "it executed and is still armed" +(`proven_since`'s whole test) plus the accuracy a dispatch counter cannot see. +Both arms take that measurement in `provision.arm_aot`, so banking it on the +arm makes adopted and self-minted cells symmetrical rather than special-casing +either. + +Two directions, and both must hold: + +* an adopted cell with a standing parity verdict is PROVEN — armed, target + installed, serving compiled, and the pass is announced + (`cell_numerics phase=serving_proof`); +* a cell with no verdict, or one whose artifact REVOKED after it was measured, + is still disarmed — and says which half was missing + (`cell_numerics phase=proof_absent`). A cell that fails numerics never + reaches this gate at all: `arm_aot` unwraps it. + +The gate itself is never stubbed. Part A drives the REAL `provision.arm_aot` +against a real packed artifact (the pgw#868 rig, whose ONE substitution is the +AOTI `.so`); part B drives the REAL `ensure_setup` warmup proof, faking only +the download and the arming policy — the same seam `test_aot_boot_proof_gap_ +pgw735.py` uses. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import msgspec +import pytest + +import gen_worker +import gen_worker.executor as ex_mod +from gen_worker import RequestContext, Resources, Slot, endpoint, worker_function +from gen_worker import activity, aot_serve, cell_adopt, compile_cache, fleet_cells +from gen_worker import numerics_ladder +from gen_worker.api.decorators import Compile +from gen_worker.executor import Executor +from gen_worker.models.refs import normalize_model_ref +from gen_worker.pb import worker_scheduler_pb2 as pb +from gen_worker.registry import extract_specs + +# The rig of the REAL arm path (real artifact, real gate, real ladder). +import test_numerics_gate_pgw868 as rig868 # noqa: E402 +from test_numerics_gate_pgw868 import ROWS, ProbePackage, arm, entry_name # noqa: E402 + +#: pgw#868's fixtures, re-exported so this module collects them: `declared` +#: registers the real export declaration the probe feed is built from, and +#: `events` captures the typed rows. +declared = rig868.declared +events = rig868.events + +FAMILY = "micro-diffusion" + + +# --------------------------------------------------------------------------- +# PART A — the parity verdict is BANKED on the arm that earned it +# --------------------------------------------------------------------------- + + +def test_a_faithful_arm_banks_its_parity_verdict( + tmp_path, monkeypatch, declared, events): + """The measurement the pod already paid for survives the function that + took it. Pre-fix it was announced and dropped, which is why the warmup + gate had nothing to read.""" + packages = {entry_name(h, w): ProbePackage() for h, w in ROWS} + pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) + + assert outcome.armed is True + proof = aot_serve.numerics_proof(pipeline) + assert proof is not None, "the arm banked no parity verdict" + assert proof.axes == len(ROWS) + assert proof.cell_key == "cell868" + assert proof.verdict == numerics_ladder.VERDICT_HEALTHY + assert proof.worst_cosine == pytest.approx(1.0, abs=1e-6) + assert proof.worst_entry in {entry_name(h, w) for h, w in ROWS} + + +def test_a_gray_band_arm_banks_its_verdict_as_degraded( + tmp_path, monkeypatch, declared, events): + """A cell inside the declared gray band ARMS and confesses. It therefore + serves, so it must carry a proof too — with the rung it actually reached, + not a rounded-up one.""" + packages = {entry_name(h, w): ProbePackage(cosine=0.997) for h, w in ROWS} + pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) + + assert outcome.armed is True + proof = aot_serve.numerics_proof(pipeline) + assert proof is not None + assert proof.verdict == numerics_ladder.VERDICT_DEGRADED + assert proof.worst_cosine == pytest.approx(0.997, abs=1e-3) + + +def test_a_refused_cell_banks_nothing(tmp_path, monkeypatch, declared, events): + """THE fail-closed half: a cell below its floor does not arm, so there is + no marker, no proof, and nothing downstream can be carried by one.""" + packages = {entry_name(h, w): ProbePackage(cosine=0.99) for h, w in ROWS} + pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) + + assert outcome.armed is False + assert aot_serve.is_armed(pipeline) is False + assert aot_serve.numerics_proof(pipeline) is None + assert aot_serve.numerics_measured(pipeline) is False + + +def test_the_proof_cannot_outlive_the_arm_it_is_about( + tmp_path, monkeypatch, declared, events): + """Scoped to the exact wrap that earned it: unwrap drops the marker, and a + revoked target keeps the record while withdrawing the PROOF — the two are + different facts and the disarm names which one it saw.""" + packages = {entry_name(h, w): ProbePackage() for h, w in ROWS} + pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) + assert outcome.armed is True + + # A revoked target: measured, but no longer serving anything. + for row in aot_serve.armed_targets(pipeline).values(): + row["state"]["failed"] = True + assert aot_serve.numerics_proof(pipeline) is None + assert aot_serve.numerics_measured(pipeline) is True + + aot_serve.unwrap(pipeline) + assert aot_serve.numerics_measured(pipeline) is False + assert aot_serve.record_numerics_proof( + pipeline, aot_serve.NumericsProof("k", 1, "e", 1.0, "healthy")) is False + + +# --------------------------------------------------------------------------- +# PART B — the boot warmup proof gate, through the real `ensure_setup` +# --------------------------------------------------------------------------- + + +class GenIn(msgspec.Struct): + prompt: str = "warm" + num_inference_steps: int = 2 + + +class Out(msgspec.Struct): + y: str = "ok" + + +class _Unet: + """The compile target: a module-shaped object with a bound forward.""" + + def __init__(self) -> None: + self.forwards = 0 + + def forward(self, *args: Any, **kwargs: Any) -> None: + self.forwards += 1 + + +class _Pipe: + def __init__(self) -> None: + self.unet = _Unet() + + +#: per-test wiring the endpoint instance reads at setup/handler time. +RIG: Dict[str, Any] = {} + + +@endpoint( + models={"pipeline": Slot(str)}, + resources=Resources(gpu=True), + compile=Compile(family=FAMILY, targets=("unet",), text_len=0, + shapes=((256, 256),)), +) +class AdoptedFamily: + def setup(self, pipeline: str) -> None: + self.pipe = _Pipe() + RIG["pipe"] = self.pipe + gen_worker.arm_compile(self.pipe) + + @worker_function() + def generate(self, ctx: RequestContext, p: GenIn) -> Out: + # The pod shape: the boot warmup runs, and its payload lands on no + # packaged entry of the adopted cell — so the artifact's own execution + # counter does not move. `RIG["dispatch"]` models the OTHER arm, where + # a warm dispatch does land. + if RIG.get("dispatch"): + marker = getattr(self.pipe, aot_serve._MARKER_ATTR, None) + if marker is not None: + marker["state"]["successful_calls"] += 1 + return Out() + + +#: The three dispatches the pgw#868 gate itself made at arm time. `aot_proof_ +#: before` is snapshotted AFTER the arm, so these prove nothing to a gate that +#: only counts — which is exactly the pod's reading. +PROBE_CALLS = 3 + + +def _fake_adopt_arm(key: str, ref: str, *, bank: bool, revoke: bool = False): + """A fleet policy standing in for §4.27 boot-adopt: arm the resolved cell + on the unet, run the parity gate (or not), and hand back the adopted + identity. Everything from `ArmOutcome` to the boot proof runs REAL.""" + + def _enable(pipe: Any, cfg: Any, cache_dir: Any, artifact: Any, + publisher: Any = None, **_kw: Any) -> "fleet_cells.ArmOutcome": + unet = pipe.unet + state = {"successful_calls": PROBE_CALLS, "failed": False, + "original": unet.forward} + marker = {"module": unet, "state": state, "meta": {"cell_key": key}} + setattr(unet, aot_serve._MARKER_ATTR, marker) + setattr(pipe, aot_serve._MARKER_ATTR, marker) + aot_serve.note_aot_key(key) + if bank: + assert aot_serve.record_numerics_proof(pipe, aot_serve.NumericsProof( + cell_key=key, axes=3, worst_entry="unet/h=256,w=256", + worst_cosine=1.0, verdict=numerics_ladder.VERDICT_HEALTHY, + elapsed_ms=158)) + if revoke: + state["failed"] = True + adopted = fleet_cells.SelfMint( + family=FAMILY, cell_key=key, ref=ref, + snapshot_digest="blake3:" + "ab" * 32, + artifact=Path(cache_dir or ".") / "cell.tar") + return fleet_cells.ArmOutcome(armed=True, self_mint=adopted) + + return _enable + + +def _rig(monkeypatch: pytest.MonkeyPatch, *, seed: str, bank: bool, + revoke: bool = False, dispatch: bool = False) -> Tuple[str, str]: + RIG.clear() + RIG["dispatch"] = dispatch + key = "ck1-" + (seed * 56)[:56] + ref = f"root/family-{FAMILY}#{key}" + monkeypatch.setattr( + fleet_cells, "enable_compiled", + _fake_adopt_arm(key, ref, bank=bank, revoke=revoke)) + return key, ref + + +@pytest.fixture +def spy(monkeypatch: pytest.MonkeyPatch) -> List[Tuple[str, str, str]]: + said: List[Tuple[str, str, str]] = [] + real = activity.emit_event + + def _spy(kind: str, detail: str = "", **kw: Any) -> Any: + said.append((kind, str(kw.get("phase", "")), detail)) + try: + return real(kind, detail, **kw) + except Exception: + return None + + monkeypatch.setattr(activity, "emit_event", _spy) + monkeypatch.setattr(ex_mod.activity_mod, "emit_event", _spy) + return said + + +def _boot(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ref: str = "acme/micro-diffusion:prod") -> Tuple[Executor, Any]: + sent: List[pb.WorkerMessage] = [] + + async def _send(msg: pb.WorkerMessage) -> None: + sent.append(msg) + + ex = Executor(extract_specs(AdoptedFamily), _send) + ex.store._cache_dir = tmp_path / "cas" + + async def _fake_download(target: str, **_kw: Any) -> Path: + p = tmp_path / target.replace("/", "_").replace(":", "_") + p.mkdir(parents=True, exist_ok=True) + return p + + monkeypatch.setattr(ex_mod, "ensure_local", _fake_download) + + from gen_worker import dispatch as dispatch_mod + + spec = ex.specs["generate"] + eff = ex._dispatched_spec( + spec, {"pipeline": dispatch_mod.SlotOrder(ref=ref, components=())}) + snaps = {normalize_model_ref(ref): pb.Snapshot( + digest="d1" * 16, + files=[pb.SnapshotFile( + path="model.safetensors", size_bytes=5, blake3="cd" * 32, + url="http://r2.invalid/presigned")])} + asyncio.run(ex.ensure_setup(eff, snaps)) + return ex, eff + + +def _phases(said: List[Tuple[str, str, str]], kind: str) -> List[str]: + return [phase for k, phase, _d in said if k == kind] + + +def test_an_adopted_cell_with_a_standing_parity_proof_STAYS_ARMED( + tmp_path, monkeypatch, spy): + """THE RED. Pre-fix: `unexercised` -> `unproven` -> unwrap -> quarantine, + `functions=()`, `target_applicability_incomplete`, eager for life. The + artifact was verified at cos=1.00000 on this very pod first.""" + _key, ref = _rig(monkeypatch, seed="a", bank=True) + ex, eff = _boot(tmp_path, monkeypatch) + pipe = RIG["pipe"] + + assert aot_serve.is_armed(pipe) is True, ( + "the pod threw away a cell it had just verified against eager") + assert getattr(pipe.unet, aot_serve._MARKER_ATTR, None) is not None + assert compile_cache.cell_proven_in_process(ref) + assert not compile_cache.cell_quarantined_in_process(ref) + + # ...and the arm is DISPATCHABLE, which is the half the pod actually lost: + # a proof that leaves `function_proofs` empty installs no target and the + # boot still ends `boot_ended_uncompiled`. + rec = ex._classes[eff.instance_key] + assert rec.compile_targets, "proven and still no installed compile target" + target = next(iter(rec.compile_targets.values())) + assert "generate" in target.function_names + assert rec.eager_posture == "" + assert cell_adopt.EagerPhase.ARMED_TARGET_UNRESOLVED.value not in _phases( + spy, "serve_eager_posture") + assert "target_applicability_incomplete" not in _phases( + spy, "serve_eager_posture") + assert ex._served_execution_lane(eff).endswith("+compiled") + + # An unannounced pass is indistinguishable from a gate that never ran. + rows = [(phase, detail) for kind, phase, detail in spy + if kind == activity.KIND_CELL_NUMERICS] + assert [p for p, _d in rows] == [numerics_ladder.PHASE_SERVING_PROOF], rows + assert "cos=1.00000" in rows[0][1] + assert "axes=3" in rows[0][1] + + +def test_an_arm_with_NO_warm_dispatch_STILL_SERVES_and_withholds_its_publish( + tmp_path, monkeypatch, spy): + """THE DELETED BARRIER, in the direction that used to be its whole point. + + Nothing dispatched through this artifact and nothing measured it, so this + pod holds no evidence either way — and under the serve-first ruling that is + NOT a verdict. The cell serves; the first real request is the proof; a + cell-attributable failure revokes it in-request. What absence of evidence + still decides is PUBLISHING: serving optimistically costs this pod one + eager fallback, publishing an unverified cell costs every pod that adopts + it.""" + _key, ref = _rig(monkeypatch, seed="b", bank=False) + ex, eff = _boot(tmp_path, monkeypatch) + pipe = RIG["pipe"] + + assert aot_serve.is_armed(pipe) is True + assert getattr(pipe.unet, aot_serve._MARKER_ATTR, None) is not None + assert not compile_cache.cell_quarantined_in_process(ref) + # ...but nothing proved it either: the publish gate reads this. + assert not compile_cache.cell_proven_in_process(ref) + rec = ex._classes[eff.instance_key] + assert rec.compile_targets, "the arm stands, so its target must be installed" + + # LOUD, at the decision point — an unannounced posture is indistinguishable + # from a gate that never ran. + rows = [(phase, detail) for kind, phase, detail in spy + if kind == activity.KIND_CELL_NUMERICS] + assert [p for p, _d in rows] == [numerics_ladder.PHASE_PROOF_ABSENT], rows + assert "STAYS ARMED" in rows[0][1] + assert "no parity verdict was banked" in rows[0][1] + + +def test_a_REVOKED_artifact_is_not_carried_by_the_verdict_it_once_passed( + tmp_path, monkeypatch, spy): + """`proven_since`'s second half, applied to the parity proof: an artifact + that was measured and then revoked has proven nothing. The arm is already + gone (the wrapper revoked it), so what this pins is that the stale verdict + does not vouch for it — the publish stays withheld.""" + _key, ref = _rig(monkeypatch, seed="c", bank=True, revoke=True) + ex, eff = _boot(tmp_path, monkeypatch) + pipe = RIG["pipe"] + + assert aot_serve.is_armed(pipe) is False + assert not compile_cache.cell_proven_in_process(ref) + rows = [(phase, detail) for kind, phase, detail in spy + if kind == activity.KIND_CELL_NUMERICS] + assert [p for p, _d in rows] == [numerics_ladder.PHASE_PROOF_ABSENT], rows + assert "revoked since it was measured" in rows[0][1] + + +# --------------------------------------------------------------------------- +# PART C — try-serve: the first real request is the proof, and what it blames +# --------------------------------------------------------------------------- + + +class _Runner: + """One entry's artifact runner: counts calls, raises what it is told to.""" + + def __init__(self, exc: Exception | None = None) -> None: + self.calls = 0 + self.exc = exc + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + self.calls += 1 + if self.exc is not None: + raise self.exc + return "compiled" + + +class _Module: + def __init__(self) -> None: + self.eager_calls = 0 + + def forward(self, *args: Any, **kwargs: Any) -> Any: + self.eager_calls += 1 + return "eager" + + +def _wrapped(runner: _Runner, spy: List[Tuple[str, str, str]]) -> Any: + module = _Module() + aot_serve.wrap_module( + module, runner, {"family": FAMILY, "cell_key": "ck1-x"}, + attr="forward", target="unet") + return module + + +def test_a_cell_that_FAILS_at_serve_answers_the_request_eager_and_stays_down( + spy, monkeypatch): + """Paul's clause 2, exactly: the tenant whose request hit the bad cell + still gets a correct answer, the disarm is STICKY for the process, and the + revocation reaches the scheduler state through the wrapper's own callback. + Nothing here is stubbed — this is the shipping wrapper.""" + runner = _Runner(RuntimeError("aoti: illegal memory access")) + module = _wrapped(runner, spy) + revoked: List[str] = [] + module._cozy_aot["state"]["failure_callback"] = revoked.append + + assert module.forward(1) == "eager", "the request must not raise" + assert module.eager_calls == 1 + assert revoked and "illegal memory access" in revoked[0] + + # Sticky: no retry loop against a broken artifact. + assert module.forward(2) == "eager" + assert runner.calls == 1, "a revoked artifact was called again" + assert module._cozy_aot["state"]["failed"] is True + + +def test_a_TRANSIENT_OOM_is_not_the_cells_fault_and_does_not_disarm_it( + spy, monkeypatch): + """Honest attribution, which is what makes serve-first safe: allocator + exhaustion is a fact about the CARD at this instant (a sibling load, a + rotation), not about the artifact. Condemning the cell for it would retire + a correct one on the first busy moment and re-mint it on the replacement + pod. The request is still answered.""" + runner = _Runner(RuntimeError("CUDA out of memory. Tried to allocate 2 GiB")) + module = _wrapped(runner, spy) + revoked: List[str] = [] + module._cozy_aot["state"]["failure_callback"] = revoked.append + + assert module.forward(1) == "eager" + assert revoked == [], "a transient OOM revoked the cell" + assert module._cozy_aot["state"]["failed"] is False + + # ...and the artifact is still the serving lane once the pressure passes. + runner.exc = None + assert module.forward(2) == "compiled" + assert [(k, p) for k, p, _d in spy if k == "aot_serve_oom"] == [ + ("aot_serve_oom", "cuda_oom")] + + +def test_a_warm_DISPATCH_still_proves_first_and_says_nothing_extra( + tmp_path, monkeypatch, spy): + """Symmetry, from the other side. The self-mint arm reaches its proof by a + warm dispatch and is untouched by this issue: when a dispatch lands, it is + the proof, and no parity row is emitted.""" + _key, ref = _rig(monkeypatch, seed="d", bank=True, dispatch=True) + ex, eff = _boot(tmp_path, monkeypatch) + pipe = RIG["pipe"] + + assert aot_serve.is_armed(pipe) is True + assert aot_serve.execution_count(pipe) > PROBE_CALLS + assert compile_cache.cell_proven_in_process(ref) + rec = ex._classes[eff.instance_key] + assert rec.compile_targets + assert [phase for kind, phase, _d in spy + if kind == activity.KIND_CELL_NUMERICS] == [] diff --git a/tests/test_aot_boot_proof_gap_pgw735.py b/tests/test_aot_boot_proof_gap_pgw735.py index 3e8593b7e..73843f097 100644 --- a/tests/test_aot_boot_proof_gap_pgw735.py +++ b/tests/test_aot_boot_proof_gap_pgw735.py @@ -12,11 +12,19 @@ 1. an exercised pure-AOT arm is PROVEN — stays armed, cell recorded proven in-process (the pgw#637 registry); - 2. an unexercised pure-AOT arm is DISARMED to true eager — artifact - unwrapped (original forward restored), identity quarantined, no - active selection kept; - 3. the same disarm on the MANDATORY (w8a8) lane rides the pgw#672 - degrade-to-eager posture without killing the boot. + 2. an unexercised pure-AOT arm KEEPS SERVING and banks no proof; + 3. the same on the MANDATORY (w8a8) lane does not kill the boot. + +**Rows 2 and 3 were the opposite assertion until pgw#1141** (Paul's ruling, +2026-08-11): an unexercised arm was unwrapped, quarantined and dropped. That +barrier is deleted — an ADOPTED cell arms before setup, so nothing has +dispatched through it by construction, and two real pods threw away artifacts +they had just verified at `cos=1.00000` because of it. The fail-closed property +this file was written to defend now lives where it belongs: the pgw#868 +numerics gate refuses a cell that does not reproduce eager, and a +cell-attributable failure at SERVE time revokes the arm in-request. What an +absent measurement still decides is the PUBLISH — which is what rows 2 and 3 +now pin. """ from __future__ import annotations @@ -194,28 +202,29 @@ def test_exercised_pure_aot_arm_is_proven_at_boot( assert not compile_cache.cell_quarantined_in_process(ref) -def test_unexercised_pure_aot_arm_disarms_to_true_eager( +def test_unexercised_pure_aot_arm_keeps_serving_and_banks_no_proof( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: + """pgw#1141: absence of a dispatch is not a verdict about the artifact. + The arm stands and the first real request is the proof; the cell is not + recorded proven, which is what the publish gate reads.""" _key, ref = _rig(monkeypatch, seed="b", exercise=False) ex = _executor(tmp_path, monkeypatch) _boot(ex) pipe = RIG["pipe"] - # The gap's fail-open half: pre-fix the arm stayed installed unproven. - assert not aot_serve.is_armed(pipe) - assert getattr(pipe.unet, aot_serve._MARKER_ATTR, None) is None - assert pipe.unet.forward.__func__ is _Unet.forward - assert compile_cache.cell_quarantined_in_process(ref) + assert aot_serve.is_armed(pipe) + assert getattr(pipe.unet, aot_serve._MARKER_ATTR, None) is not None + assert not compile_cache.cell_quarantined_in_process(ref) assert not compile_cache.cell_proven_in_process(ref) -def test_mandatory_execution_lane_disarm_degrades_without_killing_the_boot( +def test_mandatory_execution_lane_without_a_dispatch_does_not_kill_the_boot( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: _key, ref = _rig(monkeypatch, seed="c", exercise=False, weight_lane="w8a8-lora64") ex = _executor(tmp_path, monkeypatch) - _boot(ex) # pgw#672: degrade, never a load failure + _boot(ex) # pgw#672: never a load failure pipe = RIG["pipe"] - assert not aot_serve.is_armed(pipe) - assert compile_cache.cell_quarantined_in_process(ref) + assert aot_serve.is_armed(pipe) + assert not compile_cache.cell_proven_in_process(ref) From 30af52a162315a1c466f58d84b30ec9b34f4501b Mon Sep 17 00:00:00 2001 From: Paul Fidika Date: Tue, 11 Aug 2026 14:03:40 -0600 Subject: [PATCH 2/4] =?UTF-8?q?pgw#1141=20=C2=A74.32:=20quality=20is=20pro?= =?UTF-8?q?ven=20at=20MINT,=20never=20at=20adoption=20=E2=80=94=20the=20ad?= =?UTF-8?q?opt-side=20gate=20moves=20to=20the=20minting=20pod=20and=20turn?= =?UTF-8?q?s=20strict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paul refined §4.31 after the first commit here. Every numerics failure this platform ever caught (a baked `conv_out.bias`, timestep dtype scars) was an AUTHOR defect in endpoint code or config, so the check belongs to the author, not to every pod that adopts the result: re-measuring per boot taxes the fleet forever for one author's one-time mistake. * **Adoption runs no quality gate.** `provision.arm_aot` gains `verify_numerics`, defaulting to FALSE — materialize, arm, serve. The three adopt routes through `_arm_exported_cell` (hub-ordered/boot-adopt, the local store, an in-process finalized cell) all take the default. * **The mint-time gate is REAL and STRICT, and this commit is what makes it either.** It runs on exactly one path — `fleet_cells.adopt_delegated_mint` -> `arm_aot(verify_numerics=True)` — which is the arm `publish_self_mint` requires before it can ship anything, so a refusal there unwraps, serves eager, emits `self_mint_abort` to the hub and publishes nothing. §4.32 wants identical-or-refuse, so `gate_cell_numerics(strict=True)` now refuses the gray band too: a DEGRADED cell is one an adopter can never re-check. FINDING, reported rather than assumed: §4.32 says this gate "exists — pgw#1058 admission". It did not. pgw#1058 is the admission-CONTRACT identity check (an entry's generated input guards against its declared manifest), at mint and at arm; the only eager-vs-compiled parity measurement in the tree was `models/provision.gate_cell_numerics`, and its only caller was the ADOPT arm. Deleting it without moving it would have left the platform with zero parity detection anywhere. `numerics_probe` was built for exactly this move ("`measure_axis` takes callables rather than a pipeline precisely so the same measurement can run mint-side"), so it is a call-site change, not a new mechanism. * **The superseded ledger is gone.** `aot_serve.NumericsProof` / `record_numerics_proof` / `numerics_proof` / `numerics_measured` were built for the previous design (bank the adopt verdict so the warm gate could read it). With adoption measuring nothing they have no job, so they are deleted rather than left as a surface a future lane would wire back up. * **The sticky de-arm now reaches the INSTALL.** A revoked exported artifact no longer gets a compile target installed — otherwise the wire would say `serving_mode=aot_cell` on a pipeline whose every call runs eager, which is the class of lie pgw#1082/#1093 spent two pods closing. The old disarm sweep hid this case by unwrapping first; serve-first reaches it, so it is named. Safety without re-measurement, stated correctly (an earlier draft of this argument was wrong and Paul caught it — cells NEVER carry checkpoint hashes; a ck1 key is graph x envelope x sm x toolchain and one cell serves every checkpoint of the architecture, which is the point of reuse): (a) the cell is compiled CODE and weights flow through it as data (call inputs and arm-time-bound constants), so a mint-time parity proof proves the FUNCTION and transfers to any checkpoint that function accepts; (b) the one way that breaks — a weight VALUE baked into the artifact — is structurally fenced fail-closed by the constant-folding fence (0.100.0), not policed by measurement; (c) a checkpoint that changes the COMPUTATION (a config flag that alters the traced graph) hashes to a different graph, hence a different key, hence no match at all. Tests: `test_adopted_cell_warm_proof_pgw1141.py` is restructured into the four things that now have to be true — the mint gate ships or refuses (strict), an ADOPTION arms a cell that same gate would have refused and emits no verdict at all (parametrized over both failing rungs, deliberately: the point is not that adoption is lucky, it is that adoption does not ASK), the setup warmup keeps an undispatched arm and its aliases, and try-serve de-arms on the cell's fault while a transient OOM does neither. `test_numerics_gate_pgw868.py` keeps every row it had, now driving the MINT arm, with its gray-band row flipped to "confesses AND refuses to publish". `test_aot_local_mint_pgw1096.py`'s stub asserts the adopt path does NOT ask for the gate rather than absorbing the new keyword — a `**kwargs` shim there would have gone on reading green if the tax came back. --- changelog.d/pgw1141.md | 2 +- src/gen_worker/aot_serve.py | 65 ---- src/gen_worker/executor.py | 105 +++--- src/gen_worker/fleet_cells.py | 21 +- src/gen_worker/models/provision.py | 94 +++-- src/gen_worker/numerics_ladder.py | 20 +- tests/test_adopted_cell_warm_proof_pgw1141.py | 321 ++++++++++-------- tests/test_aot_local_mint_pgw1096.py | 9 +- tests/test_numerics_gate_pgw868.py | 29 +- 9 files changed, 331 insertions(+), 335 deletions(-) diff --git a/changelog.d/pgw1141.md b/changelog.d/pgw1141.md index 30cc127af..e122b3b5d 100644 --- a/changelog.d/pgw1141.md +++ b/changelog.d/pgw1141.md @@ -1 +1 @@ -- **pgw#1141: a boot-adopted cell SERVES — the setup warmup no longer disarms an exported artifact for want of a dispatch it could not have had.** Measured identically on two real pods (RTX 4000 Ada, 0.106.0): `boot_adopt=hit` -> materialize -> `cell_numerics cos=1.00000 ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored that same artifact `unexercised`, folded it into `unproven`, set `function_proofs[id]=set()` and called `aot_serve.unwrap` / `compile_cache.unwrap`. The pod served eager for life and published nothing, while the SELF-MINT arm on the same wheel, card and release served `+compiled` — because a mint's warmup drives its own capture and therefore dispatches, and an adopt arms BEFORE setup and therefore cannot. The gate was destroying evidence stronger than the evidence it demanded. Paul's ruling (2026-08-11): *"skip the warmup/arm check, so we can serve right away… try to serve, and if an error is encountered and it's the cause of the cell, de-arm the cell and serve eager instead. If our cell is correct this adds zero cost."* An AOTI artifact is ahead-of-time machine code for this exact sm/toolchain — the first call is full speed, and the warm pass never made it faster, it only checked it. On the exported lane the warm ledger now gates NOTHING: the arm stands, `_install_compile_targets` permits the whole advertised contract instead of `function_proofs` (which produced `functions=()` -> `target_applicability_incomplete` -> `armed_target_unresolved`, pgw#1093's token on its boot-adopt twin), and correctness is carried by the three detectors that can actually see a fault — the pgw#868 numerics gate refuses a cell that does not reproduce eager (the only detector for a cell that runs cleanly and returns a WRONG image), a cell-attributable failure at serve time answers that request eager and revokes the arm stickily and typed, and an out-of-envelope shape is refused by name per request. Publishing to the fleet stays evidence-gated: an artifact with neither a warm dispatch nor a standing parity verdict serves, but withholds its cell. **The DYNAMO lane keeps its ledger** — a dynamo arm that does not serve its cell RECOMPILES (correct output, silently slower, no exception to catch, and no numerics gate on that lane), so its per-class cache-hit proof is the only detector in existence. **Boot cost deleted with it:** `tracing` now means a capture is being driven or a dynamo arm is live, so a boot adopting an exported cell runs the collapsed EAGER warm plan instead of the full class cross-product — for sdxl that is 18 full generates per handler (9 aspect buckets x 2 guidance classes) reduced to 2; the pgw#654/ie#546 canary priced the same machinery at ~30-minute first boots. The parity verdict is banked on the arm that earned it (`aot_serve.NumericsProof`, scoped to the marker so `unwrap` drops it and a revoked target withdraws it), and serve-time attribution is honest: a CUDA OOM is a fact about the card at that instant, so it serves the request eager and leaves the cell armed rather than condemning a correct artifact on the first busy moment. Accepted tail risk, written down rather than hidden: an artifact pathological enough to corrupt the CUDA context is not recoverable in-request. `tests/test_adopted_cell_warm_proof_pgw1141.py` drives the real `provision.arm_aot` (real packed artifact, real gate, real ladder), the real `ensure_setup` warmup proof, and the real serving wrapper — 6 rows red before the fix, both directions plus the OOM attribution. +- **pgw#1141 (DESIGN-RULINGS §4.31 + §4.32): a boot-adopted cell materializes, arms and SERVES — the warmup proof barrier and the adopt-side quality gate are both deleted.** Measured identically on two real pods (RTX 4000 Ada, 0.106.0): `boot_adopt=hit` -> materialize -> `cell_numerics cos=1.00000 ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored that same artifact `unexercised`, folded it into `unproven`, wrote `function_proofs[id]=set()` and unwrapped it; `functions=()` made `target_applicability_incomplete` and `armed_target_unresolved` inevitable and the pod served eager for life, publishing nothing. The SELF-MINT arm was healthy on the same wheel, card and release — a mint's warmup DRIVES its own capture and therefore dispatches, while an adopt arms *before* setup and by construction cannot. **§4.31 (Paul):** *"skip the warmup / arm check, so we can serve right away … try to serve, if an error is encountered, and it's the cause of the cell, de-arm the cell, and serve eager instead. If our cell is correct this adds zero cost."* An AOTI `.pt2` is ahead-of-time machine code for this exact sm x toolchain: the first call is full speed, and the warm pass never made a cell faster, it only checked it. **§4.32** then moved the quality question to where the defect is: every failure that gate ever caught (a baked `conv_out.bias`, timestep dtype scars) was an AUTHOR defect in endpoint code or config, and re-measuring on every adopter taxed the fleet forever for one author's one-time mistake. So on the exported lane the warm ledger gates nothing (`_install_compile_targets` permits the whole advertised contract; `function_proofs` survives as telemetry), and **adoption is materialize -> arm -> serve with no quality gate at all**. The pgw#868 probe now runs on exactly ONE path — `fleet_cells.adopt_delegated_mint` -> `provision.arm_aot(verify_numerics=True)`, the pod that just compiled the bytes, before `publish_self_mint` can ship them — and it is **strict: identical or refuse, no DEGRADED-publish band**, because an adopter runs no gate that could re-check what ships. Safety without re-measurement is CONSTRUCTION, not checkpoint identity (a `ck1` key is graph x envelope x sm x toolchain and carries no checkpoint hash — one cell serves every checkpoint of the architecture): the cell is compiled CODE and weights flow through it as data, so a mint-time parity proof proves the FUNCTION; a weight value baked into the artifact is fenced fail-closed by the constant-folding fence (0.100.0); and a checkpoint that changes the COMPUTATION hashes to a different graph, hence a different key, hence no match. **Failure handling carries the weight now:** a cell-attributable failure answers that request eager, de-arms sticky and types the cause; a **CUDA OOM is not the cell's fault** (a sibling load, a rotation) so it serves eager and leaves the cell armed rather than condemning a correct artifact on the first busy moment; and the sticky de-arm now reaches the target install, so a revoked artifact can no longer advertise `aot_cell` on a pipeline whose every call runs eager. The **dynamo lane keeps its cache-hit ledger** — a dynamo arm that does not serve its cell RECOMPILES (correct output, silently slower, no exception to catch and no numerics gate on that lane), so it is the only detector in existence there. **Boot cost deleted with the barrier:** `tracing` now means a capture is being driven or a dynamo arm is live, so a boot adopting an exported cell runs the collapsed EAGER warm plan instead of the full class cross-product — for sdxl that is 18 full generates per handler (9 aspect buckets x 2 guidance classes) reduced to 2; the pgw#654/ie#546 canary priced the same machinery at ~30-minute first boots. Accepted residual, on the record: an artifact pathological enough to corrupt the CUDA context is not recoverable in-request. `tests/test_adopted_cell_warm_proof_pgw1141.py` drives the real `provision.arm_aot` (real packed artifact, real gate, real ladder), the real `ensure_setup`, and the real serving wrapper — nothing stubbed, red on master in both directions, including an adoption that arms a cell the mint gate would have refused. diff --git a/src/gen_worker/aot_serve.py b/src/gen_worker/aot_serve.py index 2ea737a7b..d1798ef86 100644 --- a/src/gen_worker/aot_serve.py +++ b/src/gen_worker/aot_serve.py @@ -2958,67 +2958,6 @@ def served_entry_calls(pipeline: Any) -> Dict[str, int]: return out -@dataclass(frozen=True) -class NumericsProof: - """What the pgw#868 gate MEASURED on THIS arm, on THIS pod (pgw#1141). - - The gate already runs the artifact — every packaged entry, through its own - runner, on the entry's own declared feed — against the eager forward it - replaces. That is an execution proof and an accuracy proof in one, and it - was discarded the moment it was announced, so the setup warmup then scored - the same artifact ``unexercised`` (no dispatch during ITS window) and - destroyed it: two real pods resolved, materialized and verified a cell at - ``cos=1.00000`` on 3/3 axes and served eager for life. - - Kept on the arm's own marker, so it is scoped to exactly the wrap that - earned it: :func:`unwrap` drops the marker and a re-arm replaces it, which - means a proof can never outlive the artifact it is about. - """ - - cell_key: str - axes: int - worst_entry: str - worst_cosine: float - verdict: str - elapsed_ms: int = 0 - - def __str__(self) -> str: - return (f"cell_numerics axes={self.axes} worst={self.worst_entry} " - f"cos={self.worst_cosine:.5f} {self.verdict} " - f"key={self.cell_key or '?'} took={self.elapsed_ms}ms") - - -def record_numerics_proof(pipeline: Any, proof: NumericsProof) -> bool: - """Bank the parity verdict on the live arm. False when nothing is armed.""" - marker = getattr(pipeline, _MARKER_ATTR, None) - if not isinstance(marker, dict) or not is_armed(pipeline): - return False - marker["numerics"] = proof - return True - - -def numerics_measured(pipeline: Any) -> bool: - """A parity verdict was banked on this marker, whether or not the cell is - still armed — the discriminator between "nobody measured it" and "it was - measured and then revoked", which are different defects.""" - marker = getattr(pipeline, _MARKER_ATTR, None) or {} - return isinstance(marker.get("numerics"), NumericsProof) - - -def numerics_proof(pipeline: Any) -> Optional[NumericsProof]: - """The parity proof this pod took on the CURRENTLY armed artifact. - - ``None`` unless a proof was banked AND the cell is still armed — the same - two-halves rule as :func:`proven_since`, for the same reason: an artifact - that ran and then revoked has not proven anything. - """ - marker = getattr(pipeline, _MARKER_ATTR, None) or {} - proof = marker.get("numerics") - if not isinstance(proof, NumericsProof) or not is_armed(pipeline): - return None - return proof - - def proven_since(pipeline: Any, before: int) -> bool: """The exported lane's ADOPTION PROOF (pgw#735). @@ -3161,10 +3100,6 @@ def unwrap(pipeline: Any) -> bool: "load_and_wrap", "marshal_positional", "note_aot_key", - "NumericsProof", - "numerics_measured", - "numerics_proof", - "record_numerics_proof", "pack", "range_digest", "resident_constants", diff --git a/src/gen_worker/executor.py b/src/gen_worker/executor.py index d7404c364..740be50ee 100644 --- a/src/gen_worker/executor.py +++ b/src/gen_worker/executor.py @@ -4713,6 +4713,26 @@ def _install_compile_targets( exported_arm = bool( active_selection is not None and aot_serve.is_aot_ref(active_selection.ref)) + if exported_arm and not aot_serve.is_armed(pipeline): + # pgw#1141: the sticky de-arm reaches the INSTALL. The artifact + # revoked itself (a failed target, a constants fault) before + # any guard was bound to hear it, so installing its target + # would advertise `serving_mode=aot_cell` on a pipeline whose + # every call now runs eager — the wire lie pgw#1082/#1093 spent + # two pods closing. Under the old barrier the disarm sweep hid + # this case by unwrapping first; serve-first reaches it, so it + # is named here. + detail = ( + f"{type(pipeline).__name__} owning slots " + f"{sorted(candidate.slots)} holds a REVOKED exported cell " + f"({active_selection.ref if active_selection else '?'}): " + f"the artifact de-armed itself during boot, so every call " + f"serves eager and no compiled target may advertise it") + logger.warning("compile target omitted for %s: %s", + spec.name, detail) + self._note_eager_posture( + rec, cell_adopt.EagerPhase.COMPILED_DEGRADED.value, detail) + continue permitted_names = ( contract_names if exported_arm else function_proofs[id(pipeline)] @@ -7514,56 +7534,18 @@ async def run_warmup() -> Tuple[int, Dict[int, set[str]], str]: if proved_sel is not None: compile_cache.record_cell_proven(proved_sel.ref) continue - # pgw#1141: THE second proof, and on a boot-ADOPTED - # cell it is the only one that can exist. The arm - # happens before setup, so nothing has dispatched yet - # by construction — but the pgw#868 gate has already - # run EVERY packaged entry through its own runner - # against the eager forward it replaces, on this pod, - # on these weights, on this card. That is a superset of - # what `proven_since` asks (it executed, and it is - # still armed) plus the accuracy the dispatch counter - # cannot see. Scoring it `unexercised` destroyed - # artifacts verified at cos=1.00000 on two real pods, - # and left the SELF-MINT arm — which reaches its proof - # by a warm dispatch — as the only way to serve - # compiled. Both arms take the same measurement in - # `provision.arm_aot`, so reading it here is what makes - # adopted and self-minted cells symmetrical rather - # than a special case for one of them. - parity = aot_serve.numerics_proof(pipe) - if parity is None: - arm_without_dispatch[id(pipe)] = ( - "artifact revoked since it was measured" - if aot_serve.numerics_measured(pipe) - else "no parity verdict was banked on this arm") - unexercised.append(candidate) - continue - proven += 1 - # `function_proofs` is deliberately NOT narrowed to - # `spec.name` here: the parity verdict covers every - # entry the cell PACKAGES, i.e. the whole contract its - # key advertises, so the install falls through to - # `contract_names`. A class the cell does not carry is - # refused BY NAME at ingress and served eager per - # request (pgw#844), which is the backstop that lets - # this attribution be contract-wide. - proved_sel = inj.active_compile_artifacts.get(id(pipe)) - if proved_sel is not None: - compile_cache.record_cell_proven(proved_sel.ref) - logger.info( - "compile object (slots=%s) proven by its own " - "numerics parity (%s); the boot warmup landed no " - "dispatch on it (pgw#1141)", - sorted(candidate.slots), parity) - activity_mod.emit_event( - activity_mod.KIND_CELL_NUMERICS, - f"{spec.name}: the boot warmup dispatched no call " - f"through this armed cell, and its own parity " - f"measurement on THIS pod carries the proof " - f"instead — {parity}", - phase=numerics_ladder.PHASE_SERVING_PROOF, - ) + # pgw#1141 / §4.31 + §4.32: an adopted cell arms BEFORE + # setup, so no dispatch can have landed by now, and + # nothing measures it here either — quality was proven + # once, on the pod that MINTED it, and adoption runs no + # quality gate. The absence of a dispatch is therefore + # not a verdict: the arm stands, the first real request + # is the proof, and a cell-attributable failure de-arms + # it in-request. + arm_without_dispatch[id(pipe)] = ( + "adoption runs no quality gate (§4.32) — this cell " + "was proven at its mint") + unexercised.append(candidate) continue before = proof_before.get(id(pipe)) if before is None: @@ -7643,11 +7625,10 @@ def _confess_arm_without_dispatch( activity_mod.emit_event( activity_mod.KIND_CELL_NUMERICS, f"{spec.name}: the exported cell on slots " - f"{sorted(candidate.slots)} took no warm dispatch and " - f"{reason} — it STAYS ARMED and serves; a " - f"cell-attributable failure revokes it in-request, and " - f"it publishes nothing to the fleet", - phase=numerics_ladder.PHASE_PROOF_ABSENT, + f"{sorted(candidate.slots)} took no warm dispatch — " + f"{reason}. It STAYS ARMED and serves; a " + f"cell-attributable failure revokes it in-request", + phase=numerics_ladder.PHASE_ARMED_UNDISPATCHED, ) # pgw#1141 (Paul's ruling, 2026-08-11), and it is a DELETION: @@ -7837,12 +7818,14 @@ def _confess_arm_without_dispatch( "cell-attributable failure revokes it in-request " "(pgw#1141)", sorted(candidate.slots)) _confess_arm_without_dispatch(candidate) - if aot_serve.numerics_proof(pipe) is None: - # Nothing on this pod has evidence this artifact - # works: it may serve (try-serve decides), but it - # must not be PUBLISHED to pods that cannot - # re-derive that verdict. - self._abandon_pending_mint(inj, pipe) + # NOTE the publish is NOT withheld here any more. §4.32 + # moves that authority to the mint-time gate on this + # same pod (`fleet_cells.adopt_delegated_mint` -> + # `provision.arm_aot(verify_numerics=True)`), which runs + # the freshly compiled artifact against the eager + # forward it was traced from and refuses to publish + # anything that is not identical. A warm dispatch count + # decides nothing at either end. continue mandatory = pipeline_weight_lane(pipe).startswith( _MANDATORY_EXECUTION_LANES) diff --git a/src/gen_worker/fleet_cells.py b/src/gen_worker/fleet_cells.py index 803c708fb..ef21017bd 100644 --- a/src/gen_worker/fleet_cells.py +++ b/src/gen_worker/fleet_cells.py @@ -1812,9 +1812,17 @@ def local_keep_reason(publisher: Optional[CellPublisher]) -> str: def _arm_exported_cell( pipe: Any, cfg: Any, cache_dir: Optional[Path], bucket: int, artifact: Path, arm_key: Optional[ArmIdentity], + *, verify_numerics: bool = False, ) -> Tuple[bool, Optional[Dict[str, Any]], Tuple[str, str]]: """THE gate every cell this machine produced for itself must pass. + ``verify_numerics`` (§4.32) is set by ONE caller — + :func:`adopt_delegated_mint`, the pod that just minted these bytes and is + about to publish them. The other three routes here are ADOPTIONS of bytes + already proven at their own mint (an in-process finalized cell, the local + store's, a fresh child mint being re-armed), and adoption runs no quality + gate. + pgw#1096 extracted this from :func:`adopt_delegated_mint` so the two self-produced sources — a child process's fresh mint, and this machine's OWN local store (§4.28) — pass through ONE gate rather than two that agree @@ -1874,7 +1882,9 @@ def _arm_exported_cell( f"the cell (stamped key {stamped}) does not describe this " f"runtime: {divergence}")) try: - outcome = provision.arm_aot(pipe, cfg, cache_dir, artifact, bucket, meta) + outcome = provision.arm_aot( + pipe, cfg, cache_dir, artifact, bucket, meta, + verify_numerics=verify_numerics) if outcome: return True, meta, ("", "") refusal = (outcome.reason or "unclassified_arm_refusal", @@ -2069,10 +2079,17 @@ def adopt_delegated_mint( # the reasons this call site used to compute inline are unchanged, and the # $2.72 lesson (attempt 26: a 36/36 mint refused by three events that all # said "could not adopt") is kept there rather than repeated here. + # §4.32 THE MINT-TIME GATE, and this is the only site that arms it: this + # process compiled these bytes and is about to publish them to every pod + # that will ever adopt this key. It runs the freshly compiled artifact and + # the eager forward it was traced from on the same feed, and it is STRICT — + # identical or refuse, no gray band — because an adopter runs no gate that + # could re-check what ships. A refusal below unwraps, serves eager, emits + # `self_mint_abort` to the hub and publishes nothing. armed, meta, refusal = _arm_exported_cell( pipe, pending.cfg, pending.cache_dir, int(getattr(pending.cfg, "lora_bucket", 0) or 0), - pending.target, pending.arm_key) + pending.target, pending.arm_key, verify_numerics=True) if not armed: reason, detail = refusal # pgw#999: `phase` is the countable column, so it carries the CLASS — diff --git a/src/gen_worker/models/provision.py b/src/gen_worker/models/provision.py index c42a37a15..a84bab98d 100644 --- a/src/gen_worker/models/provision.py +++ b/src/gen_worker/models/provision.py @@ -309,10 +309,17 @@ def arm_aot( pipe: Any, cfg: Any, cache_dir: Optional[Path], artifact: Path, bucket: int, meta: Optional[Dict[str, Any]] = None, *, expected: "Optional[ExpectedIdentity]" = None, + verify_numerics: bool = False, ) -> AdoptOutcome: """Arm ONE exported ``.pt2`` cell on ``pipe``. The whole AOT arm, in one place, for every source of such an artifact. + ``verify_numerics`` (DESIGN-RULINGS §4.32, pgw#1141) runs the parity gate, + and exactly ONE caller sets it: the pod that MINTED these bytes, before it + publishes them. Adoption runs no quality gate at all — see + :func:`gate_cell_numerics` for why re-measuring an adopted cell was taxing + every adopter forever for an author's one-time mistake. + pgw#805 extracted this from :func:`enable_compiled`'s kind dispatch: a cell this pod MINTED ITSELF has to arm through exactly the same gates a hub-delivered one does (that is the point of the delegated split — a @@ -435,7 +442,9 @@ def arm_aot( + f" — {lifted_install_error}]".strip(), outcome.identity) if outcome.armed: - if gate_cell_numerics(pipe, cfg): + # §4.32: quality is proven at MINT and in author CI, never at adoption. + # An adopting pod materializes, arms and serves. + if not verify_numerics or gate_cell_numerics(pipe, cfg, strict=True): return outcome # A refused cell is UNARMED, not merely reported: the whole point is # that it must not serve. Staying eager is the ordinary miss policy @@ -451,9 +460,10 @@ def arm_aot( aot_serve.unwrap(pipe) outcome = AdoptOutcome.miss( "numerics_refused", - f"family={meta.get('family')} key={meta.get('cell_key')}: armed, " - f"then UNARMED by the numerics gate — this pod serves eager " - f"(pgw#868)", + f"family={meta.get('family')} key={meta.get('cell_key')}: this pod " + f"MINTED these bytes and they do not reproduce the eager forward " + f"they were traced from — nothing is published and this pod serves " + f"eager (pgw#868, §4.32)", outcome.identity) if lifted_installed: from . import lora_lifted @@ -462,26 +472,47 @@ def arm_aot( return outcome -def gate_cell_numerics(pipe: Any, cfg: Any) -> bool: +def gate_cell_numerics(pipe: Any, cfg: Any, *, strict: bool = False) -> bool: """THE numerics gate (pgw#868): does this cell reproduce the eager forward - it is about to replace? Returns False when it must not serve. - - Placed at ADOPT time because a hub-delivered cell never re-enters mint — - the arm is the only point every cell passes through, and it is the point - that decides whether the artifact serves. `numerics_probe.measure_axis` - takes callables rather than a pipeline precisely so the same measurement - can also run mint-side later as a cheaper early catch; what must not exist - is a mint-only gate, which cannot protect an adopting pod whose weights, - lane and card differ. - - Three outcomes, all of them typed rows on the wire: - - * HEALTHY -> `cell_numerics phase=checked`, arms silently-but-recorded. + it replaces? Returns False when it must not serve — and, on the only path + that runs it, when it must not be PUBLISHED. + + **It runs on the MINTING pod, and nowhere else (DESIGN-RULINGS §4.32).** + It used to run at every ADOPT, on the reasoning that a mint-only gate + "cannot protect an adopting pod whose weights, lane and card differ". Paul + overruled that: every failure this gate has ever caught (a baked + `conv_out.bias`, timestep dtype scars) was an AUTHOR defect in endpoint + code or config, and re-measuring on every adopter taxes the whole fleet + forever for one author's one-time mistake. It is not the consumer's job to + catch the author's bugs. + + What makes adoption safe without re-measuring is CONSTRUCTION, not identity + — a ck1 key is graph x envelope x sm x toolchain and carries NO checkpoint + hash, deliberately, so one cell serves every checkpoint of the + architecture: + + * the cell is compiled CODE; weights flow through it as data (call inputs + and arm-time-bound constants), so a mint-time parity proof proves the + FUNCTION and transfers to any checkpoint that function accepts; + * the one way that breaks — a weight VALUE baked into the artifact — is + structurally fenced fail-closed by the constant-folding fence (0.100.0), + not policed by measurement; + * a checkpoint that changes the COMPUTATION (a config flag that alters the + traced graph) hashes to a different graph, hence a different key, hence + no match at all. The graph axis protects there, not a checkpoint digest. + + ``strict`` is what the mint path passes, and §4.32 requires it: identical + or refuse, with no DEGRADED-publish band. A cell that lands in the gray + band is one an ADOPTER can never re-check, so shipping it would export an + unmeasured degradation to every pod that pulls it. + + Outcomes, all typed rows on the wire: + + * HEALTHY -> `cell_numerics phase=checked`; the mint arms and publishes. The pass is announced deliberately: an unannounced pass is indistinguishable from a gate that never ran, which is this program's signature failure. - * DEGRADED -> the ladder's `phase=degraded` row; the cell ARMS and - confesses. + * DEGRADED -> `phase=degraded`, and under ``strict`` it REFUSES. * DESTROYED / unmeasurable -> refuse. `numerics_ladder.gate` raises the typed refusal below the floor; a probe that could not be TAKEN is refused on its own `phase=unmeasurable`, because "nobody could ask" is @@ -513,7 +544,6 @@ def gate_cell_numerics(pipe: Any, cfg: Any) -> bool: else "no_axis_measured"), f"{report.context()} | unmeasured: {rows}") comparison = report.comparison() - worst = report.worst() try: numerics_ladder.gate( comparison, @@ -521,19 +551,15 @@ def gate_cell_numerics(pipe: Any, cfg: Any) -> bool: refuse=lambda detail, worst_row: numerics_probe.CellNumericsRefused( detail, worst_row), context=report.context()) - # pgw#1141: the verdict is BANKED on the arm, not merely announced. - # This measurement executed every packaged entry through its own - # runner and matched eager — so the setup warmup, which scores an - # exported arm by "did a dispatch land during MY window", has a proof - # to read instead of destroying the artifact that carries it. - aot_serve.record_numerics_proof(pipe, aot_serve.NumericsProof( - cell_key=str(report.cell_key or ""), - axes=len(report.verdicts), - worst_entry=(worst.axis.name if worst is not None else ""), - worst_cosine=(comparison.cosine if comparison is not None else 0.0), - verdict=(comparison.verdict if comparison is not None else ""), - elapsed_ms=report.elapsed_ms, - )) + if strict and comparison is not None and not comparison.healthy: + # §4.32: identical or refuse. The ladder already emitted the + # `degraded` row above, so the confession is on the wire; what + # changes here is that the bytes do not ship. + logger.error( + "aot mint: REFUSING to publish %s — the cell it just compiled " + "lands in the gray band (%s), and an adopter runs no gate that " + "could re-check it (§4.32)", family or "cell", comparison.verdict) + return False if comparison is not None and comparison.healthy: activity_mod.emit_event( activity_mod.KIND_CELL_NUMERICS, diff --git a/src/gen_worker/numerics_ladder.py b/src/gen_worker/numerics_ladder.py index 84f849326..c2e277d9a 100644 --- a/src/gen_worker/numerics_ladder.py +++ b/src/gen_worker/numerics_ladder.py @@ -53,16 +53,13 @@ PHASE_DEGRADED = "degraded" PHASE_REFUSED = "refused" -#: pgw#1141 — the two phases the BOOT WARMUP proof gate reports against a -#: parity verdict, on this same kind so one query (`?kind=cell_numerics`) -#: answers what happened to every cell that armed on a pod. -#: ``serving_proof``: the boot warmup landed no dispatch on an armed exported -#: cell and the pod's OWN parity measurement carried the proof instead (an -#: unannounced pass is indistinguishable from a gate that never ran). -#: ``proof_absent``: no dispatch AND no standing parity verdict — the artifact -#: is disarmed, and this row says which half was missing. -PHASE_SERVING_PROOF = "serving_proof" -PHASE_PROOF_ABSENT = "proof_absent" +#: pgw#1141 (§4.31/§4.32) — an armed exported cell took no warm dispatch this +#: boot, which is the NORMAL state of every adopted cell (the arm precedes +#: setup) and is no longer a verdict about it. Carried on this kind so one +#: query (`?kind=cell_numerics`) still answers what happened to every cell that +#: armed on a pod, and emitted because an unannounced posture is +#: indistinguishable from a gate that never ran. +PHASE_ARMED_UNDISPATCHED = "armed_undispatched" def worst_of(*verdicts: str) -> str: @@ -449,8 +446,7 @@ def declared_thresholds(cfg: Any) -> Thresholds: __all__ = [ "DEFAULT_THRESHOLDS", "NUMERICS_FLOOR", "NUMERICS_RETENTION_FLOOR", "NUMERICS_WARN", - "PHASE_DEGRADED", "PHASE_PROOF_ABSENT", "PHASE_REFUSED", - "PHASE_SERVING_PROOF", + "PHASE_ARMED_UNDISPATCHED", "PHASE_DEGRADED", "PHASE_REFUSED", "VERDICT_DEGRADED", "VERDICT_DESTROYED", "VERDICT_HEALTHY", "Comparison", "RowStat", "Thresholds", "compare_outputs", "declared_thresholds", "flatten_outputs", "gate", diff --git a/tests/test_adopted_cell_warm_proof_pgw1141.py b/tests/test_adopted_cell_warm_proof_pgw1141.py index b73840ef1..483e720ba 100644 --- a/tests/test_adopted_cell_warm_proof_pgw1141.py +++ b/tests/test_adopted_cell_warm_proof_pgw1141.py @@ -1,39 +1,53 @@ -"""pgw#1141: a boot-ADOPTED cell carries its own proof, and the setup warmup -proof gate must READ it instead of destroying the artifact that holds it. +"""pgw#1141 (DESIGN-RULINGS §4.31 + §4.32): a boot-ADOPTED cell materializes, +arms and SERVES. No warmup barrier, and no quality gate at adoption. MEASURED, twice, on two real pods (RTX 4000 Ada, gen-worker 0.106.0, hub `7ae35d54a2`): `boot_adopt=hit` -> materialize -> `cell_numerics cos=1.00000 -ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored the +ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored that same artifact `unexercised` (it dispatched nothing through it — the adopt arms -BEFORE setup, so by construction nothing has), folded it into `unproven`, -`function_proofs[id]=set()`, `aot_serve.unwrap`, `compile_cache.unwrap`. The -pod served eager for life and published nothing. The self-mint arm on the SAME -wheel, card and release served `+compiled` with an empty `fallback_reason`, -because it reaches its proof through a warm dispatch. - -So the gate was destroying evidence STRONGER than the evidence it demands. The -pgw#868 numerics gate runs every packaged entry through its own runner against -the eager forward it replaces — that is "it executed and is still armed" -(`proven_since`'s whole test) plus the accuracy a dispatch counter cannot see. -Both arms take that measurement in `provision.arm_aot`, so banking it on the -arm makes adopted and self-minted cells symmetrical rather than special-casing -either. - -Two directions, and both must hold: - -* an adopted cell with a standing parity verdict is PROVEN — armed, target - installed, serving compiled, and the pass is announced - (`cell_numerics phase=serving_proof`); -* a cell with no verdict, or one whose artifact REVOKED after it was measured, - is still disarmed — and says which half was missing - (`cell_numerics phase=proof_absent`). A cell that fails numerics never - reaches this gate at all: `arm_aot` unwraps it. - -The gate itself is never stubbed. Part A drives the REAL `provision.arm_aot` -against a real packed artifact (the pgw#868 rig, whose ONE substitution is the -AOTI `.so`); part B drives the REAL `ensure_setup` warmup proof, faking only -the download and the arming policy — the same seam `test_aot_boot_proof_gap_ -pgw735.py` uses. +BEFORE setup, so by construction nothing has), folded it into `unproven`, wrote +`function_proofs[id]=set()` and unwrapped it. `functions=()` then made the +target omission (`target_applicability_incomplete`) and the orphan report +(`armed_target_unresolved`) inevitable, and the pod served eager for life. The +SELF-MINT arm was healthy on the same wheel, card and release, because a mint's +warmup DRIVES its own capture and therefore dispatches. + +§4.31 deleted the barrier: *"skip the warmup / arm check, so we can serve right +away … try to serve, if an error is encountered, and it's the cause of the +cell, de-arm the cell, and serve eager instead."* + +§4.32 then deleted the adopt-side numerics re-check too, and moved the quality +question to where the defect is: every failure that gate ever caught (a baked +`conv_out.bias`, timestep dtype scars) was an AUTHOR defect in endpoint code or +config. Re-measuring on every adopter taxes the fleet forever for one author's +one-time mistake. Adoption is materialize -> arm -> serve; the gate runs ONCE, +on the pod that minted the bytes, before they are published, and it is STRICT. + +Safety without re-measurement comes from CONSTRUCTION, not from checkpoint +identity (a `ck1` key is graph x envelope x sm x toolchain and carries no +checkpoint hash — one cell serves every checkpoint of the architecture, which +is the whole point of reuse): the cell is compiled CODE and weights flow +through it as data, so a mint-time parity proof proves the FUNCTION; a weight +value baked into the artifact is fenced fail-closed by the constant-folding +fence; and a checkpoint that changes the COMPUTATION hashes to a different +graph, hence a different key, hence no match. + +What this file pins, in four parts: + +A. the MINT gate — strict, identical-or-refuse, and it is the thing that + decides whether bytes ship; +B. ADOPTION — a cell that would FAIL that gate still arms and serves, and no + quality row is emitted at all, because nobody measured it; +C. the executor's setup warmup — an armed cell with no dispatch keeps its arm, + its target and its aliases; +D. try-serve — a cell-attributable failure answers the request eager and + de-arms sticky; a transient OOM does neither. + +Nothing is stubbed. Parts A and B drive the REAL `provision.arm_aot` against a +real packed artifact (the pgw#868 rig, whose ONE substitution is the AOTI +`.so`); part C drives the REAL `ensure_setup`, faking only the download and the +arming policy (the seam `test_aot_boot_proof_gap_pgw735.py` uses); part D +drives the real serving wrapper. """ from __future__ import annotations @@ -70,78 +84,106 @@ # --------------------------------------------------------------------------- -# PART A — the parity verdict is BANKED on the arm that earned it +# PART A — the MINT gate: strict, and it decides whether bytes ship # --------------------------------------------------------------------------- -def test_a_faithful_arm_banks_its_parity_verdict( +def test_the_MINTING_pod_proves_its_own_bytes_before_they_ship( tmp_path, monkeypatch, declared, events): - """The measurement the pod already paid for survives the function that - took it. Pre-fix it was announced and dropped, which is why the warmup - gate had nothing to read.""" + """§4.32 item 2. The pod that compiled the artifact runs it against the + eager forward it was traced from, on the same feed, and only then does the + arm succeed — which is what `adopt_delegated_mint` needs to be true before + `publish_self_mint` can ship anything.""" packages = {entry_name(h, w): ProbePackage() for h, w in ROWS} - pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) + pipeline, _module, outcome = arm( + tmp_path, monkeypatch, declared, packages, verify_numerics=True) assert outcome.armed is True - proof = aot_serve.numerics_proof(pipeline) - assert proof is not None, "the arm banked no parity verdict" - assert proof.axes == len(ROWS) - assert proof.cell_key == "cell868" - assert proof.verdict == numerics_ladder.VERDICT_HEALTHY - assert proof.worst_cosine == pytest.approx(1.0, abs=1e-6) - assert proof.worst_entry in {entry_name(h, w) for h, w in ROWS} + assert aot_serve.is_armed(pipeline) is True + rows = [(p, d) for k, d, p in events if k == activity.KIND_CELL_NUMERICS] + assert [p for p, _d in rows] == ["checked"], rows + assert "axes=2/2" in rows[0][1] -def test_a_gray_band_arm_banks_its_verdict_as_degraded( +def test_the_MINT_gate_is_strict_a_gray_band_cell_does_not_ship( tmp_path, monkeypatch, declared, events): - """A cell inside the declared gray band ARMS and confesses. It therefore - serves, so it must carry a proof too — with the rung it actually reached, - not a rounded-up one.""" + """§4.32: identical or refuse, no DEGRADED-publish band. An adopter runs no + gate that could re-check what ships, so a gray-band publish would export an + unmeasured degradation to every pod that pulls the key.""" packages = {entry_name(h, w): ProbePackage(cosine=0.997) for h, w in ROWS} - pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) + pipeline, _module, outcome = arm( + tmp_path, monkeypatch, declared, packages, verify_numerics=True) - assert outcome.armed is True - proof = aot_serve.numerics_proof(pipeline) - assert proof is not None - assert proof.verdict == numerics_ladder.VERDICT_DEGRADED - assert proof.worst_cosine == pytest.approx(0.997, abs=1e-3) + assert outcome.armed is False, "a gray-band cell was published to the fleet" + assert aot_serve.is_armed(pipeline) is False + assert outcome.reason == "numerics_refused" + assert "nothing is published" in outcome.detail + # It still CONFESSES — a fleet-wide rate is only countable from rows. + assert [p for k, _d, p in events + if k == activity.KIND_CELL_NUMERICS] == ["degraded"] -def test_a_refused_cell_banks_nothing(tmp_path, monkeypatch, declared, events): - """THE fail-closed half: a cell below its floor does not arm, so there is - no marker, no proof, and nothing downstream can be carried by one.""" +def test_the_MINT_gate_refuses_a_cell_below_its_floor( + tmp_path, monkeypatch, declared, events): + """Unchanged by both rulings, and the reason the gate still exists at all: + try-serve catches ERRORS, never wrong OUTPUT. A cell that runs cleanly and + renders a bad image raises nothing.""" packages = {entry_name(h, w): ProbePackage(cosine=0.99) for h, w in ROWS} - pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) + pipeline, _module, outcome = arm( + tmp_path, monkeypatch, declared, packages, verify_numerics=True) assert outcome.armed is False assert aot_serve.is_armed(pipeline) is False - assert aot_serve.numerics_proof(pipeline) is None - assert aot_serve.numerics_measured(pipeline) is False + assert [p for k, _d, p in events + if k == activity.KIND_CELL_NUMERICS] == ["refused"] + + +# --------------------------------------------------------------------------- +# PART B — ADOPTION runs no quality gate at all +# --------------------------------------------------------------------------- -def test_the_proof_cannot_outlive_the_arm_it_is_about( +@pytest.mark.parametrize("cosine,label", [ + (0.99, "below the floor"), + (0.997, "inside the gray band"), +]) +def test_ADOPTION_arms_a_cell_the_mint_gate_would_have_REFUSED( + tmp_path, monkeypatch, declared, events, cosine, label): + """THE §4.32 RED, and it fails on master twice over: the adopt path used to + run this gate and would have unwrapped both of these cells. + + Adoption is materialize -> arm -> serve. The bytes were proven once, at + their mint; re-proving them on every adopter taxes the fleet forever for an + author's one-time mistake, and it is the tax that made a pod throw away a + cell it had just verified at `cos=1.00000`. Deliberately parametrized over + a cell that would FAIL: the point is not that adoption is lucky, it is that + adoption does not ASK.""" + packages = {entry_name(h, w): ProbePackage(cosine=cosine) for h, w in ROWS} + pipeline, _module, outcome = arm( + tmp_path, monkeypatch, declared, packages, verify_numerics=False) + + assert outcome.armed is True, f"an adopting pod re-judged a cell {label}" + assert aot_serve.is_armed(pipeline) is True + assert [k for k, _d, _p in events if k == activity.KIND_CELL_NUMERICS] == [], ( + "adoption emitted a quality verdict, so it ran a quality gate") + + +def test_ADOPTION_is_the_default_so_a_new_arm_path_cannot_inherit_the_tax( tmp_path, monkeypatch, declared, events): - """Scoped to the exact wrap that earned it: unwrap drops the marker, and a - revoked target keeps the record while withdrawing the PROOF — the two are - different facts and the disarm names which one it saw.""" - packages = {entry_name(h, w): ProbePackage() for h, w in ROWS} - pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) - assert outcome.armed is True + """`verify_numerics` defaults to False, and that direction is deliberate: + the ONE caller that measures is the mint. A future arm path that forgets + the flag adopts (correct); one that forgets it in the other direction would + have silently re-imposed the per-adopter cost this issue deleted.""" + import inspect - # A revoked target: measured, but no longer serving anything. - for row in aot_serve.armed_targets(pipeline).values(): - row["state"]["failed"] = True - assert aot_serve.numerics_proof(pipeline) is None - assert aot_serve.numerics_measured(pipeline) is True + from gen_worker.models import provision - aot_serve.unwrap(pipeline) - assert aot_serve.numerics_measured(pipeline) is False - assert aot_serve.record_numerics_proof( - pipeline, aot_serve.NumericsProof("k", 1, "e", 1.0, "healthy")) is False + sig = inspect.signature(provision.arm_aot) + assert sig.parameters["verify_numerics"].default is False # --------------------------------------------------------------------------- -# PART B — the boot warmup proof gate, through the real `ensure_setup` +# PART C — the setup warmup, through the real `ensure_setup` # --------------------------------------------------------------------------- @@ -198,16 +240,16 @@ def generate(self, ctx: RequestContext, p: GenIn) -> Out: return Out() -#: The three dispatches the pgw#868 gate itself made at arm time. `aot_proof_ -#: before` is snapshotted AFTER the arm, so these prove nothing to a gate that -#: only counts — which is exactly the pod's reading. +#: Calls the artifact had already served before setup opened. Non-zero because +#: `aot_proof_before` is snapshotted AFTER the arm, so a counter-only gate can +#: never see them — which is exactly the pod's reading. PROBE_CALLS = 3 -def _fake_adopt_arm(key: str, ref: str, *, bank: bool, revoke: bool = False): +def _fake_adopt_arm(key: str, ref: str, *, revoke: bool = False): """A fleet policy standing in for §4.27 boot-adopt: arm the resolved cell - on the unet, run the parity gate (or not), and hand back the adopted - identity. Everything from `ArmOutcome` to the boot proof runs REAL.""" + on the unet and hand back the adopted identity. Everything from + `ArmOutcome` to the setup warmup runs REAL.""" def _enable(pipe: Any, cfg: Any, cache_dir: Any, artifact: Any, publisher: Any = None, **_kw: Any) -> "fleet_cells.ArmOutcome": @@ -218,11 +260,6 @@ def _enable(pipe: Any, cfg: Any, cache_dir: Any, artifact: Any, setattr(unet, aot_serve._MARKER_ATTR, marker) setattr(pipe, aot_serve._MARKER_ATTR, marker) aot_serve.note_aot_key(key) - if bank: - assert aot_serve.record_numerics_proof(pipe, aot_serve.NumericsProof( - cell_key=key, axes=3, worst_entry="unet/h=256,w=256", - worst_cosine=1.0, verdict=numerics_ladder.VERDICT_HEALTHY, - elapsed_ms=158)) if revoke: state["failed"] = True adopted = fleet_cells.SelfMint( @@ -234,7 +271,7 @@ def _enable(pipe: Any, cfg: Any, cache_dir: Any, artifact: Any, return _enable -def _rig(monkeypatch: pytest.MonkeyPatch, *, seed: str, bank: bool, +def _rig(monkeypatch: pytest.MonkeyPatch, *, seed: str, revoke: bool = False, dispatch: bool = False) -> Tuple[str, str]: RIG.clear() RIG["dispatch"] = dispatch @@ -242,7 +279,7 @@ def _rig(monkeypatch: pytest.MonkeyPatch, *, seed: str, bank: bool, ref = f"root/family-{FAMILY}#{key}" monkeypatch.setattr( fleet_cells, "enable_compiled", - _fake_adopt_arm(key, ref, bank=bank, revoke=revoke)) + _fake_adopt_arm(key, ref, revoke=revoke)) return key, ref @@ -298,26 +335,31 @@ def _phases(said: List[Tuple[str, str, str]], kind: str) -> List[str]: return [phase for k, phase, _d in said if k == kind] -def test_an_adopted_cell_with_a_standing_parity_proof_STAYS_ARMED( +def test_an_ADOPTED_cell_serves_COMPILED_immediately_after_materialize( tmp_path, monkeypatch, spy): - """THE RED. Pre-fix: `unexercised` -> `unproven` -> unwrap -> quarantine, - `functions=()`, `target_applicability_incomplete`, eager for life. The - artifact was verified at cos=1.00000 on this very pod first.""" - _key, ref = _rig(monkeypatch, seed="a", bank=True) + """THE HEADLINE RED, and on master it fails twice over — once for the + warmup barrier (§4.31) and once for the adopt-side gate (§4.32). + + The pod's own reading, reproduced: the cell arms before setup, the boot + warmup dispatches nothing through it, and pre-fix that made it + `unexercised` -> `unproven` -> unwrap -> quarantine, `functions=()`, + `target_applicability_incomplete`, `armed_target_unresolved`, eager for + life. Now: armed, dispatchable, serving compiled, with NO quality + measurement taken anywhere on this pod.""" + _key, ref = _rig(monkeypatch, seed="a") ex, eff = _boot(tmp_path, monkeypatch) pipe = RIG["pipe"] assert aot_serve.is_armed(pipe) is True, ( - "the pod threw away a cell it had just verified against eager") + "the pod threw away the cell it had just materialized") assert getattr(pipe.unet, aot_serve._MARKER_ATTR, None) is not None - assert compile_cache.cell_proven_in_process(ref) assert not compile_cache.cell_quarantined_in_process(ref) # ...and the arm is DISPATCHABLE, which is the half the pod actually lost: - # a proof that leaves `function_proofs` empty installs no target and the - # boot still ends `boot_ended_uncompiled`. + # an empty `function_proofs` installs no target and the boot still ends + # `boot_ended_uncompiled` however armed the object is. rec = ex._classes[eff.instance_key] - assert rec.compile_targets, "proven and still no installed compile target" + assert rec.compile_targets, "armed and still no installed compile target" target = next(iter(rec.compile_targets.values())) assert "generate" in target.function_names assert rec.eager_posture == "" @@ -327,66 +369,43 @@ def test_an_adopted_cell_with_a_standing_parity_proof_STAYS_ARMED( spy, "serve_eager_posture") assert ex._served_execution_lane(eff).endswith("+compiled") - # An unannounced pass is indistinguishable from a gate that never ran. - rows = [(phase, detail) for kind, phase, detail in spy - if kind == activity.KIND_CELL_NUMERICS] - assert [p for p, _d in rows] == [numerics_ladder.PHASE_SERVING_PROOF], rows - assert "cos=1.00000" in rows[0][1] - assert "axes=3" in rows[0][1] - - -def test_an_arm_with_NO_warm_dispatch_STILL_SERVES_and_withholds_its_publish( - tmp_path, monkeypatch, spy): - """THE DELETED BARRIER, in the direction that used to be its whole point. - - Nothing dispatched through this artifact and nothing measured it, so this - pod holds no evidence either way — and under the serve-first ruling that is - NOT a verdict. The cell serves; the first real request is the proof; a - cell-attributable failure revokes it in-request. What absence of evidence - still decides is PUBLISHING: serving optimistically costs this pod one - eager fallback, publishing an unverified cell costs every pod that adopts - it.""" - _key, ref = _rig(monkeypatch, seed="b", bank=False) - ex, eff = _boot(tmp_path, monkeypatch) - pipe = RIG["pipe"] - - assert aot_serve.is_armed(pipe) is True - assert getattr(pipe.unet, aot_serve._MARKER_ATTR, None) is not None - assert not compile_cache.cell_quarantined_in_process(ref) - # ...but nothing proved it either: the publish gate reads this. - assert not compile_cache.cell_proven_in_process(ref) - rec = ex._classes[eff.instance_key] - assert rec.compile_targets, "the arm stands, so its target must be installed" - - # LOUD, at the decision point — an unannounced posture is indistinguishable - # from a gate that never ran. + # §4.32: adoption measured nothing. The only cell_numerics row a boot like + # this may emit is the posture one — never a verdict. rows = [(phase, detail) for kind, phase, detail in spy if kind == activity.KIND_CELL_NUMERICS] - assert [p for p, _d in rows] == [numerics_ladder.PHASE_PROOF_ABSENT], rows + assert [p for p, _d in rows] == [ + numerics_ladder.PHASE_ARMED_UNDISPATCHED], rows assert "STAYS ARMED" in rows[0][1] - assert "no parity verdict was banked" in rows[0][1] + assert "adoption runs no quality gate" in rows[0][1] -def test_a_REVOKED_artifact_is_not_carried_by_the_verdict_it_once_passed( +def test_a_REVOKED_artifact_is_not_re_armed_by_the_setup_pass( tmp_path, monkeypatch, spy): - """`proven_since`'s second half, applied to the parity proof: an artifact - that was measured and then revoked has proven nothing. The arm is already - gone (the wrapper revoked it), so what this pins is that the stale verdict - does not vouch for it — the publish stays withheld.""" - _key, ref = _rig(monkeypatch, seed="c", bank=True, revoke=True) + """The de-arm is STICKY (§4.31). An artifact the wrapper already revoked — + it ran and failed — is not resurrected by anything downstream, and the boot + lands on eager with its target omitted rather than advertising a lane that + cannot serve.""" + _key, ref = _rig(monkeypatch, seed="c", revoke=True) ex, eff = _boot(tmp_path, monkeypatch) pipe = RIG["pipe"] assert aot_serve.is_armed(pipe) is False assert not compile_cache.cell_proven_in_process(ref) - rows = [(phase, detail) for kind, phase, detail in spy - if kind == activity.KIND_CELL_NUMERICS] - assert [p for p, _d in rows] == [numerics_ladder.PHASE_PROOF_ABSENT], rows - assert "revoked since it was measured" in rows[0][1] + rec = ex._classes[eff.instance_key] + assert not rec.compile_targets, ( + "a revoked artifact kept an installed target, so the wire would say " + "aot_cell on a pipeline whose every call runs eager") + # The revocation is NAMED on the wire, at the install that refused it. + # (`rec.eager_posture` settles on the terminal `armed_target_unresolved` + # from `_assert_armed_targets_installed`, which is the orphan report for + # the same object; both rows are emitted and neither is silence.) + assert cell_adopt.EagerPhase.COMPILED_DEGRADED.value in _phases( + spy, "serve_eager_posture") + assert rec.eager_posture # --------------------------------------------------------------------------- -# PART C — try-serve: the first real request is the proof, and what it blames +# PART D — try-serve: the first real request is the proof, and what it blames # --------------------------------------------------------------------------- @@ -467,10 +486,10 @@ def test_a_TRANSIENT_OOM_is_not_the_cells_fault_and_does_not_disarm_it( def test_a_warm_DISPATCH_still_proves_first_and_says_nothing_extra( tmp_path, monkeypatch, spy): - """Symmetry, from the other side. The self-mint arm reaches its proof by a - warm dispatch and is untouched by this issue: when a dispatch lands, it is - the proof, and no parity row is emitted.""" - _key, ref = _rig(monkeypatch, seed="d", bank=True, dispatch=True) + """Symmetry, from the other side. A warm dispatch that DOES land still + records the cell proven in-process (the pgw#637 registry the dynamo lane + reads), and emits no posture row — there is nothing to explain.""" + _key, ref = _rig(monkeypatch, seed="d", dispatch=True) ex, eff = _boot(tmp_path, monkeypatch) pipe = RIG["pipe"] diff --git a/tests/test_aot_local_mint_pgw1096.py b/tests/test_aot_local_mint_pgw1096.py index 7c559d735..d365b3048 100644 --- a/tests/test_aot_local_mint_pgw1096.py +++ b/tests/test_aot_local_mint_pgw1096.py @@ -375,7 +375,14 @@ def armable(monkeypatch: pytest.MonkeyPatch) -> List[Path]: seen: List[Path] = [] def _arm(pipe: Any, cfg: Any, cache_dir: Any, artifact: Path, - bucket: int, meta: Any = None) -> AdoptOutcome: + bucket: int, meta: Any = None, *, + verify_numerics: bool = False, **_kw: Any) -> AdoptOutcome: + # pgw#1141 / §4.32: this is an ADOPTION — the bytes were proven at + # their own mint — so it must NOT ask for the quality gate. Asserted + # rather than absorbed: a `**kwargs` stub would have swallowed the + # difference and this file would still read green if the tax came back. + assert verify_numerics is False, ( + "the local store's ADOPT path asked for the mint-time gate") seen.append(Path(artifact)) return AdoptOutcome.hit(str((meta or {}).get("cell_key") or "")) diff --git a/tests/test_numerics_gate_pgw868.py b/tests/test_numerics_gate_pgw868.py index eb0507100..40ed84e43 100644 --- a/tests/test_numerics_gate_pgw868.py +++ b/tests/test_numerics_gate_pgw868.py @@ -226,9 +226,15 @@ def events(monkeypatch: pytest.MonkeyPatch) -> List[Tuple[str, str, str]]: def arm(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, decl: Any, packages: Dict[str, ProbePackage], - meta: Dict[str, Any] | None = None) -> Tuple[Any, Any, Any]: + meta: Dict[str, Any] | None = None, + verify_numerics: bool = True) -> Tuple[Any, Any, Any]: """Drive the REAL arm path and return ``(pipeline, module, outcome)``. + ``verify_numerics=True`` is the MINT arm (pgw#1141 / DESIGN-RULINGS §4.32): + since this issue the gate runs on the pod that minted the bytes, before it + publishes them, and nowhere else. Adoption passes False and runs no gate — + `test_adopted_cell_warm_proof_pgw1141.py` owns that direction. + pgw#923: the arm returns a typed :class:`AdoptOutcome` rather than a bool, so its verdict — armed, or refused with the classified reason — is a value the caller can assert on and the executor can put on the wire. It stays @@ -244,7 +250,8 @@ def arm(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, decl: Any, module = ProbeDenoiser() pipeline = ProbePipeline(module) outcome = provision.arm_aot( - pipeline, decl, tmp_path / "cache", artifact(tmp_path, meta), 0) + pipeline, decl, tmp_path / "cache", artifact(tmp_path, meta), 0, + verify_numerics=verify_numerics) return pipeline, module, outcome @@ -285,7 +292,7 @@ def test_a_cell_below_its_declared_floor_REFUSES_TO_ARM( # numerics refusal and a second "retraction" row existed only to correct # the first. The arm returns ONE outcome, with the gate's verdict in it. assert outcome.reason == "numerics_refused" - assert "UNARMED by the numerics gate" in outcome.detail + assert "nothing is published" in outcome.detail rows = numerics_rows(events) assert rows, "a refused cell said nothing on the wire" @@ -298,15 +305,21 @@ def test_a_cell_below_its_declared_floor_REFUSES_TO_ARM( assert "0.99" in detail -def test_between_floor_and_warn_it_ARMS_and_records_the_warning( +def test_the_gray_band_CONFESSES_AND_REFUSES_TO_PUBLISH( tmp_path, monkeypatch, declared, events): - """The gray band is not a refusal and not a silence. It serves, and it - confesses — a fleet-wide rate is only countable from activity records.""" + """The gray band still confesses — a fleet-wide rate is only countable from + activity records — but since §4.32 it does not SHIP. + + pgw#1141 moved this gate to the minting pod and made it strict: identical + or refuse. A degraded cell is one an adopter can never re-check (adoption + runs no quality gate at all), so publishing it would export an unmeasured + degradation to every pod that pulls the key. Before that ruling this exact + cell armed and shipped.""" packages = {entry_name(h, w): ProbePackage(cosine=0.997) for h, w in ROWS} pipeline, _module, outcome = arm(tmp_path, monkeypatch, declared, packages) - assert outcome.armed is True, "a cell inside the declared gray band failed to arm" - assert aot.is_armed(pipeline) is True + assert outcome.armed is False, "a gray-band cell was published to the fleet" + assert aot.is_armed(pipeline) is False rows = numerics_rows(events) assert [p for _d, p in rows] == ["degraded"], rows detail = rows[0][0] From 9dd0582d0cf1f3eea5ce581ecd0d3344772e005c Mon Sep 17 00:00:00 2001 From: Paul Fidika Date: Tue, 11 Aug 2026 14:07:49 -0600 Subject: [PATCH 3/4] =?UTF-8?q?pgw#1141:=20pin=20the=20MOVE=20at=20the=20p?= =?UTF-8?q?ublish=20gate=20=E2=80=94=20a=20divergent=20cell=20is=20refused?= =?UTF-8?q?=20by=20the=20pod=20that=20minted=20it,=20end=20to=20end?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The survey that corrected my brief is right that this is a MOVE and not a delete, and $4.32's sequencing rule (at no commit may zero parity gates exist) deserves a test rather than an argument. `test_a_DIVERGENT_cell_is_not_ published_by_the_pod_that_minted_it` drives the REAL `adopt_delegated_mint` — the arm `publish_self_mint` requires — over a REAL packed cell with the REAL gate underneath, and asserts nothing ships plus the two typed rows (the ladder's `cell_numerics phase=refused` and the mint's `self_mint_abort phase=numerics_refused`). It is GREEN on master too, and the docstring says so. Master satisfied the invariant by ACCIDENT of placement: the gate lived in `arm_aot`, and a mint arms, so the mint path inherited a check aimed at adopters. This commit satisfies it on PURPOSE — aimed at the mint, with nothing left on the adopt path. The rows that separate those two worlds are the ones that go red: the gray band (master ships a degraded cell) and every adoption row. --- tests/test_adopted_cell_warm_proof_pgw1141.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/test_adopted_cell_warm_proof_pgw1141.py b/tests/test_adopted_cell_warm_proof_pgw1141.py index 483e720ba..e222d22ba 100644 --- a/tests/test_adopted_cell_warm_proof_pgw1141.py +++ b/tests/test_adopted_cell_warm_proof_pgw1141.py @@ -138,6 +138,81 @@ def test_the_MINT_gate_refuses_a_cell_below_its_floor( if k == activity.KIND_CELL_NUMERICS] == ["refused"] +def _pending(tmp_path: Path, decl: Any, publisher: Any = None): + """A real `PendingSelfMint` pointing at a real packed cell.""" + target = tmp_path / "mint" / "cell.tar.gz" + target.parent.mkdir(parents=True, exist_ok=True) + return fleet_cells.PendingSelfMint( + family=rig868.FAMILY, arm_token="arm1-" + "a" * 24, + ref=f"root/family-{rig868.FAMILY}#cell868", + cfg=decl, target=target, mint_root=tmp_path / "mint", + publisher=publisher, cache_dir=tmp_path / "cache") + + +def _delegated_mint(tmp_path, monkeypatch, decl, packages, events): + """Drive the REAL `adopt_delegated_mint` — the mint's publish gate — with + the REAL `arm_aot` and the REAL numerics gate underneath it.""" + from gen_worker import aot_serve as aot + + monkeypatch.setattr(aot, "runtime_key", lambda: dict(rig868.RUNTIME)) + monkeypatch.setattr(aot, "_entry_admission_drift", lambda *a, **k: None) + monkeypatch.setattr(aot, "_load_package", lambda path, entry="model": packages[entry]) + monkeypatch.setattr(fleet_cells, "arm_axis_divergence", lambda a, m: "") + monkeypatch.setattr(fleet_cells.activity_mod, "emit_event", + lambda kind, detail="", **kw: events.append( + (kind, detail, str(kw.get("phase", ""))))) + pending = _pending(tmp_path, decl) + built = rig868.artifact(tmp_path) + pending.target.write_bytes(Path(built).read_bytes()) + module = rig868.ProbeDenoiser() + pipeline = rig868.ProbePipeline(module) + return fleet_cells.adopt_delegated_mint(pipeline, pending, pending.target) + + +def test_a_DIVERGENT_cell_is_not_published_by_the_pod_that_minted_it( + tmp_path, monkeypatch, declared, events): + """§4.32 item 2, end to end through the REAL publish gate. + + `publish_self_mint` can only ship what `adopt_delegated_mint` marked + minted, so a refusal here IS the publish refusal. + + HONEST STATUS: this row is GREEN on master too, and that is the point of + writing it. The move is a MOVE — the property "a divergent cell does not + publish" must survive it unbroken, because §4.32's sequencing rule is that + at no commit may zero parity gates exist. Master satisfied it by accident + of placement (the gate lived in `arm_aot`, and a mint arms too, so the + mint path inherited a check that was really aimed at adopters); this commit + satisfies it on purpose, with the check aimed at the mint and nothing left + on the adopt path. The rows that go RED are the ones that distinguish those + two worlds: the gray band (master ships it), and every adopt row.""" + packages = {rig868.entry_name(h, w): ProbePackage(cosine=0.99) + for h, w in ROWS} + minted = _delegated_mint(tmp_path, monkeypatch, declared, packages, events) + + assert minted is None, "a cell that does not reproduce eager was published" + # Typed, and the hub can count it: the ladder's own refusal plus the mint's + # abort, which is the row that says nothing shipped. + assert ("refused" in [p for k, _d, p in events + if k == activity.KIND_CELL_NUMERICS]) + aborts = [(d, p) for k, d, p in events if k == "self_mint_abort"] + assert aborts, "the mint published nothing and said nothing" + assert aborts[-1][1] == "numerics_refused" + assert "nothing is published" in aborts[-1][0] + + +def test_a_FAITHFUL_cell_passes_the_mint_gate_and_is_publishable( + tmp_path, monkeypatch, declared, events): + """The control, without which the row above could pass for the wrong + reason (an unreadable envelope, a divergence gate, a missing stamp).""" + packages = {rig868.entry_name(h, w): ProbePackage() for h, w in ROWS} + minted = _delegated_mint(tmp_path, monkeypatch, declared, packages, events) + + assert minted is not None, "a faithful cell was refused by the mint gate" + assert minted.cell_key == "cell868" + assert [p for k, _d, p in events + if k == activity.KIND_CELL_NUMERICS] == ["checked"] + + # --------------------------------------------------------------------------- # PART B — ADOPTION runs no quality gate at all # --------------------------------------------------------------------------- From 01da698c9d112a230035222bd916c1fc81456f05 Mon Sep 17 00:00:00 2001 From: Paul Fidika Date: Tue, 11 Aug 2026 14:32:35 -0600 Subject: [PATCH 4/4] =?UTF-8?q?pgw#1141:=20keep=20the=20warm=20PLAN=20out?= =?UTF-8?q?=20of=20this=20PR=20=E2=80=94=20the=20arm=20fix=20does=20not=20?= =?UTF-8?q?need=20it,=20and=20the=20collapse=20is=20not=20free?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI found the price. Making `tracing` mean "a capture is being driven" collapsed an exported adopt to the eager warm plan (sdxl: 18 full generates per handler -> 2, against a pgw#654/ie#546 canary that priced the machinery at ~30-minute first boots) — but that same plan is what produces pgw#844's BOOT-TIME coverage census (`compiled_shape_coverage`, which names the declared classes a cell does not carry before any tenant meets one) and what feeds the dynamo lane's per-class cache-hit ledger, its only detector of a silent recompile. `test_partial_shape_coverage_pgw844` went red on exactly that. §4.31 deletes the warm plan as a PREREQUISITE TO ARMING, which is this issue and is untouched by the revert: the arm stands, the target installs, the cell serves. The per-class COST is a separate saving that needs its own answer for the census and its own red tests, so it is filed rather than ridden in on a P0 arm fix. The omission is written into the code at the decision point. Also fixes the three stub signatures CI caught (`test_abandoned_mint_telemetry_ pgw848`, `test_local_serve_no_publisher_pgw1127`, `test_boot_adopt_local_first_ pgw1127`): each now ASSERTS the adopt path does not ask for the mint-time gate rather than absorbing the keyword through `**kwargs` — a shim there would keep those files green if the per-adopter tax came back. --- changelog.d/pgw1141.md | 2 +- src/gen_worker/executor.py | 33 +++++++++---------- tests/test_abandoned_mint_telemetry_pgw848.py | 7 +++- tests/test_boot_adopt_local_first_pgw1127.py | 9 ++++- .../test_local_serve_no_publisher_pgw1127.py | 9 ++++- 5 files changed, 39 insertions(+), 21 deletions(-) diff --git a/changelog.d/pgw1141.md b/changelog.d/pgw1141.md index e122b3b5d..02d0886a5 100644 --- a/changelog.d/pgw1141.md +++ b/changelog.d/pgw1141.md @@ -1 +1 @@ -- **pgw#1141 (DESIGN-RULINGS §4.31 + §4.32): a boot-adopted cell materializes, arms and SERVES — the warmup proof barrier and the adopt-side quality gate are both deleted.** Measured identically on two real pods (RTX 4000 Ada, 0.106.0): `boot_adopt=hit` -> materialize -> `cell_numerics cos=1.00000 ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored that same artifact `unexercised`, folded it into `unproven`, wrote `function_proofs[id]=set()` and unwrapped it; `functions=()` made `target_applicability_incomplete` and `armed_target_unresolved` inevitable and the pod served eager for life, publishing nothing. The SELF-MINT arm was healthy on the same wheel, card and release — a mint's warmup DRIVES its own capture and therefore dispatches, while an adopt arms *before* setup and by construction cannot. **§4.31 (Paul):** *"skip the warmup / arm check, so we can serve right away … try to serve, if an error is encountered, and it's the cause of the cell, de-arm the cell, and serve eager instead. If our cell is correct this adds zero cost."* An AOTI `.pt2` is ahead-of-time machine code for this exact sm x toolchain: the first call is full speed, and the warm pass never made a cell faster, it only checked it. **§4.32** then moved the quality question to where the defect is: every failure that gate ever caught (a baked `conv_out.bias`, timestep dtype scars) was an AUTHOR defect in endpoint code or config, and re-measuring on every adopter taxed the fleet forever for one author's one-time mistake. So on the exported lane the warm ledger gates nothing (`_install_compile_targets` permits the whole advertised contract; `function_proofs` survives as telemetry), and **adoption is materialize -> arm -> serve with no quality gate at all**. The pgw#868 probe now runs on exactly ONE path — `fleet_cells.adopt_delegated_mint` -> `provision.arm_aot(verify_numerics=True)`, the pod that just compiled the bytes, before `publish_self_mint` can ship them — and it is **strict: identical or refuse, no DEGRADED-publish band**, because an adopter runs no gate that could re-check what ships. Safety without re-measurement is CONSTRUCTION, not checkpoint identity (a `ck1` key is graph x envelope x sm x toolchain and carries no checkpoint hash — one cell serves every checkpoint of the architecture): the cell is compiled CODE and weights flow through it as data, so a mint-time parity proof proves the FUNCTION; a weight value baked into the artifact is fenced fail-closed by the constant-folding fence (0.100.0); and a checkpoint that changes the COMPUTATION hashes to a different graph, hence a different key, hence no match. **Failure handling carries the weight now:** a cell-attributable failure answers that request eager, de-arms sticky and types the cause; a **CUDA OOM is not the cell's fault** (a sibling load, a rotation) so it serves eager and leaves the cell armed rather than condemning a correct artifact on the first busy moment; and the sticky de-arm now reaches the target install, so a revoked artifact can no longer advertise `aot_cell` on a pipeline whose every call runs eager. The **dynamo lane keeps its cache-hit ledger** — a dynamo arm that does not serve its cell RECOMPILES (correct output, silently slower, no exception to catch and no numerics gate on that lane), so it is the only detector in existence there. **Boot cost deleted with the barrier:** `tracing` now means a capture is being driven or a dynamo arm is live, so a boot adopting an exported cell runs the collapsed EAGER warm plan instead of the full class cross-product — for sdxl that is 18 full generates per handler (9 aspect buckets x 2 guidance classes) reduced to 2; the pgw#654/ie#546 canary priced the same machinery at ~30-minute first boots. Accepted residual, on the record: an artifact pathological enough to corrupt the CUDA context is not recoverable in-request. `tests/test_adopted_cell_warm_proof_pgw1141.py` drives the real `provision.arm_aot` (real packed artifact, real gate, real ladder), the real `ensure_setup`, and the real serving wrapper — nothing stubbed, red on master in both directions, including an adoption that arms a cell the mint gate would have refused. +- **pgw#1141 (DESIGN-RULINGS §4.31 + §4.32): a boot-adopted cell materializes, arms and SERVES — the warmup proof barrier and the adopt-side quality gate are both deleted.** Measured identically on two real pods (RTX 4000 Ada, 0.106.0): `boot_adopt=hit` -> materialize -> `cell_numerics cos=1.00000 ret=1.0000 rel_l2=0.0000` on 3/3 axes -> and then the setup warmup scored that same artifact `unexercised`, folded it into `unproven`, wrote `function_proofs[id]=set()` and unwrapped it; `functions=()` made `target_applicability_incomplete` and `armed_target_unresolved` inevitable and the pod served eager for life, publishing nothing. The SELF-MINT arm was healthy on the same wheel, card and release — a mint's warmup DRIVES its own capture and therefore dispatches, while an adopt arms *before* setup and by construction cannot. **§4.31 (Paul):** *"skip the warmup / arm check, so we can serve right away … try to serve, if an error is encountered, and it's the cause of the cell, de-arm the cell, and serve eager instead. If our cell is correct this adds zero cost."* An AOTI `.pt2` is ahead-of-time machine code for this exact sm x toolchain: the first call is full speed, and the warm pass never made a cell faster, it only checked it. **§4.32** then moved the quality question to where the defect is: every failure that gate ever caught (a baked `conv_out.bias`, timestep dtype scars) was an AUTHOR defect in endpoint code or config, and re-measuring on every adopter taxed the fleet forever for one author's one-time mistake. So on the exported lane the warm ledger gates nothing (`_install_compile_targets` permits the whole advertised contract; `function_proofs` survives as telemetry), and **adoption is materialize -> arm -> serve with no quality gate at all**. The pgw#868 probe now runs on exactly ONE path — `fleet_cells.adopt_delegated_mint` -> `provision.arm_aot(verify_numerics=True)`, the pod that just compiled the bytes, before `publish_self_mint` can ship them — and it is **strict: identical or refuse, no DEGRADED-publish band**, because an adopter runs no gate that could re-check what ships. Safety without re-measurement is CONSTRUCTION, not checkpoint identity (a `ck1` key is graph x envelope x sm x toolchain and carries no checkpoint hash — one cell serves every checkpoint of the architecture): the cell is compiled CODE and weights flow through it as data, so a mint-time parity proof proves the FUNCTION; a weight value baked into the artifact is fenced fail-closed by the constant-folding fence (0.100.0); and a checkpoint that changes the COMPUTATION hashes to a different graph, hence a different key, hence no match. **Failure handling carries the weight now:** a cell-attributable failure answers that request eager, de-arms sticky and types the cause; a **CUDA OOM is not the cell's fault** (a sibling load, a rotation) so it serves eager and leaves the cell armed rather than condemning a correct artifact on the first busy moment; and the sticky de-arm now reaches the target install, so a revoked artifact can no longer advertise `aot_cell` on a pipeline whose every call runs eager. The **dynamo lane keeps its cache-hit ledger** — a dynamo arm that does not serve its cell RECOMPILES (correct output, silently slower, no exception to catch and no numerics gate on that lane), so it is the only detector in existence there. The warm PLAN's per-class cost is deliberately left for its own change: collapsing it on an exported adopt (sdxl: 18 full generates per handler — 9 aspect buckets x 2 guidance classes — reduced to 2, against a pgw#654/ie#546 canary that priced the machinery at ~30-minute first boots) also removes pgw#844's boot-time coverage census and the dynamo lane's per-class ledger, and neither should be a rider on a P0 arm fix. Nothing about the arm depends on it. Accepted residual, on the record: an artifact pathological enough to corrupt the CUDA context is not recoverable in-request. `tests/test_adopted_cell_warm_proof_pgw1141.py` drives the real `provision.arm_aot` (real packed artifact, real gate, real ladder), the real `ensure_setup`, and the real serving wrapper — nothing stubbed, red on master in both directions, including an adoption that arms a cell the mint gate would have refused. diff --git a/src/gen_worker/executor.py b/src/gen_worker/executor.py index 740be50ee..6fb05c5a6 100644 --- a/src/gen_worker/executor.py +++ b/src/gen_worker/executor.py @@ -6405,24 +6405,23 @@ async def _run_synthesized_warmup( memory = self._warm_contract_runs.setdefault( self._warm_contract_key(spec), set()) armed_refs = tuple(armed_cell_refs) - # pgw#1141: TRACING is what makes the full class x bucket cross-product - # worth its wall time — a capture being DRIVEN (every declared graph - # must trace into it), or a DYNAMO arm, whose only detector of a - # non-serving cell is the per-class cache-hit ledger this plan feeds. + # Tracing == some artifact is armed or minting on this setup; only + # then does the full class x bucket cross-product buy anything (each + # graph must trace into the capture / prove against the cell). # - # It used to mean "some artifact is armed", which charged a boot - # ADOPTING AN EXPORTED CELL the whole cross-product (sdxl: 9 aspect - # buckets x 2 guidance classes = 18 full generates per handler) for a - # proof that no longer gates anything on that lane. An AOTI cell is - # ahead-of-time machine code for this exact sm/toolchain: the first - # call is full speed, so re-running every declared class before READY - # buys nothing the numerics gate and the in-request fallback do not - # already buy — it is pure boot latency the tenant waits through. Such - # a boot now runs the EAGER plan (one collapsed run per (function, - # guidance class)), which is the allocator/kernel warm the eager - # fallback path needs anyway. - tracing = bool(cold_proof_ids) or any( - not aot_serve.is_aot_ref(ref) for ref in armed_refs) + # pgw#1141 DELIBERATELY LEAVES THIS ALONE, and the omission is the + # decision. §4.31 deletes the warm plan as a PREREQUISITE TO ARMING — + # which is this issue — and notes that the per-class cost then buys + # nothing on an exported adopt. Collapsing it to the eager plan is a + # real saving (sdxl: 18 full generates per handler -> 2) but it is not + # free: this plan is also what produces pgw#844's BOOT-TIME coverage + # census (`compiled_shape_coverage`, which names the declared classes a + # cell does not carry before any tenant meets one) and what feeds the + # dynamo lane's per-class cache-hit ledger, its only detector of a + # silent recompile. Both deserve an answer of their own rather than a + # rider on a P0 arm fix, so the collapse is filed as its own change + # with its own red tests. Nothing about the ARM depends on it. + tracing = bool(objects) skip_ok = ( allow_contract_skip and not cold_proof_ids diff --git a/tests/test_abandoned_mint_telemetry_pgw848.py b/tests/test_abandoned_mint_telemetry_pgw848.py index 4fd3f7eee..46ac8cee9 100644 --- a/tests/test_abandoned_mint_telemetry_pgw848.py +++ b/tests/test_abandoned_mint_telemetry_pgw848.py @@ -513,8 +513,13 @@ def test_the_unchecked_announcement_is_gone_because_the_gate_landed( cfg = type("Cfg", (), {"family": "sdxl", "numerics_floor": 0.995, "numerics_warn": 0.999, "lora_bucket": 0, "targets": ()})() + # pgw#1141 / §4.32: the gate runs on the MINT arm now (adoption runs no + # quality gate at all), so this is the flag the minting pod passes. What + # this test is about is unchanged: an arm that could not be MEASURED is a + # refusal, never a silent `unchecked` announcement. assert provision.arm_aot( - object(), cfg, None, Path("cell.pt2"), 0).armed is False + object(), cfg, None, Path("cell.pt2"), 0, + verify_numerics=True).armed is False rows = [(d, p) for k, d, p in said if k == activity_mod.KIND_CELL_NUMERICS] assert rows, "an arm that could not be measured said nothing" detail, phase = rows[-1] diff --git a/tests/test_boot_adopt_local_first_pgw1127.py b/tests/test_boot_adopt_local_first_pgw1127.py index f77d28dc4..99f4fe82d 100644 --- a/tests/test_boot_adopt_local_first_pgw1127.py +++ b/tests/test_boot_adopt_local_first_pgw1127.py @@ -390,7 +390,14 @@ def armable(monkeypatch: pytest.MonkeyPatch) -> List[Path]: seen: List[Path] = [] def _arm(pipe: Any, cfg: Any, cache_dir: Any, artifact: Path, - bucket: int, expected: Any = None) -> AdoptOutcome: + bucket: int, expected: Any = None, *, + verify_numerics: bool = False, **_kw: Any) -> AdoptOutcome: + # pgw#1141 / §4.32: the local store's route is an ADOPTION — these + # bytes were proven at their own mint — so it must not ask for the + # mint-time gate. Asserted rather than absorbed: a bare `**kwargs` + # shim would keep this file green if the per-adopter tax came back. + assert verify_numerics is False, ( + "the local store's ADOPT path asked for the mint-time gate") seen.append(Path(artifact)) return AdoptOutcome.hit(KEY_A) diff --git a/tests/test_local_serve_no_publisher_pgw1127.py b/tests/test_local_serve_no_publisher_pgw1127.py index 4e7da4e2a..0f8dd6a43 100644 --- a/tests/test_local_serve_no_publisher_pgw1127.py +++ b/tests/test_local_serve_no_publisher_pgw1127.py @@ -131,7 +131,14 @@ def armable(monkeypatch: pytest.MonkeyPatch) -> List[Path]: seen: List[Path] = [] def _arm(pipe: Any, cfg: Any, cache_dir: Any, artifact: Path, - bucket: int, expected: Any = None) -> AdoptOutcome: + bucket: int, expected: Any = None, *, + verify_numerics: bool = False, **_kw: Any) -> AdoptOutcome: + # pgw#1141 / §4.32: the local store's route is an ADOPTION — these + # bytes were proven at their own mint — so it must not ask for the + # mint-time gate. Asserted rather than absorbed: a bare `**kwargs` + # shim would keep this file green if the per-adopter tax came back. + assert verify_numerics is False, ( + "the local store's ADOPT path asked for the mint-time gate") seen.append(Path(artifact)) return AdoptOutcome.hit(KEY_A)