diff --git a/changelog.d/pgw1137.md b/changelog.d/pgw1137.md new file mode 100644 index 00000000..5f2e3347 --- /dev/null +++ b/changelog.d/pgw1137.md @@ -0,0 +1 @@ +- **pgw#1137 (DESIGN-RULINGS §4.30, and pgw#1127 §7 item 6): an AOT mint on the user's OWN machine is polite — declared, never toggled.** Since pgw#1127 S1 `cozy serve` drives the real `mint_delegate` → `mint_process` → `mint_child` chain, so cozy-local now compiles on a desktop using every clamp `aot_compile_pool` has — and all of those are written for a serving pod with a co-resident TENANT, not for a machine with a co-resident HUMAN. pgw#1127 §6 recorded the evidence: **zero `os.nice` calls anywhere in `src/`**. A new `compile_posture.CompilePosture` carries the one fact that decides this, and it is **declared by the entry that knows** (`local_serve`, the only site in the tree that says `USER_MACHINE`) rather than sniffed off any of the three near-miss proxies — `local_cell_store.trust_class()` and `publisher is None` are facts about the SINK, and a rented community-cloud pod matches both while having nobody sitting at it; `worker_goals` says what a pod was bought to do, and cozy-local was not bought. It is not an env var either (§1.17: an env may carry a VALUE, not a DECISION): it rides `MintRequest` as a typed field beside `vram_cap_bytes`, and the mint child publishes it before anything sizes a pool. **CPU:** the child drops itself to `nice 19` — by the CHILD, on itself, never through a `preexec_fn`, for the fork-vs-`posix_spawn`/gRPC-`pthread_atfork` hazard `arm_parent_death_signal` already writes down; and because every production mint has been weight-free (and therefore serial, in that very process) since pgw#1080, nicing the pool's spawns would have niced a path that does not currently run. Nice is inherited, so one call covers the serial compile, the entry children when pgw#1111 restores them, inductor's compile workers and every `cc1plus` under them. `entry_workers` additionally halves its core budget and halves its ceiling (8 → 4): priority is what preserves interactivity, but it does nothing for K concurrent `cc1plus`, K inductor caches and K working sets in the page cache. **Memory:** the host-RAM reserve doubles to 8 GiB, because `MemAvailable` counts the user's page cache as free and a mint that OOMs a desktop is worse than a slow one. **Honesty:** a `cozy serve` compile now says what it is doing, that it happens once, where the cell will be kept, what it is taking (nice level, worker cap, cores) and that Ctrl-C is safe — then prints a throttled progress line every 10 s. Before this the only signals were an activity addressed to a hub cozy-local does not have and a `logger.info` inside a subprocess. **Interruptibility:** the mint child arms `PR_SET_PDEATHSIG` on a user machine — it is spawned `start_new_session=True`, so a terminal's Ctrl-C or SIGHUP never reached it and a closed terminal left a full-speed compile tree running with nobody to reap it; `aot_resume`'s cross-attempt bank already lives outside the per-attempt workdir, so nothing it finished is lost. **The serving pod is untouched**: `FLEET` is the struct default everywhere, and a pod's width is proven arithmetically identical against an independent re-implementation of the pre-issue formula over six pod shapes. Every politeness claim is individually mutation-proven RED. Rejected with reasons in `compile_posture`'s docstring: an interactivity probe (cross-platform mess, and wrong for the developer running a build in a terminal — `nice` already IS the yield), `ionice` (no stdlib wrapper, a no-op on the schedulers desktops actually ship), and a third posture for community cloud. diff --git a/src/gen_worker/aot_compile_pool.py b/src/gen_worker/aot_compile_pool.py index b9442152..d3fd9492 100644 --- a/src/gen_worker/aot_compile_pool.py +++ b/src/gen_worker/aot_compile_pool.py @@ -67,7 +67,9 @@ from . import aot_shape_hints from . import aot_compile_spans, aot_device_lock, aot_resume, env_seal -from . import mint_budget, worker_goals +from . import compile_posture, mint_budget, worker_goals +from .compile_posture import ( + USER_MACHINE_RSS_RESERVE_BYTES, CompilePosture) from .worker_goals import WorkerGoals from .postmortem import cpu_quota_cores import hashlib @@ -481,6 +483,11 @@ class PoolWidth: #: exact case Paul's ruling requires to work. serve_goal: bool = True mint_goal: bool = False + #: §4.30 / pgw#1137: whose MACHINE this is. Distinct from the goals above, + #: which say what the pod was bought to do — a K held down for a human at + #: a keyboard and a K held down for a tenant are different decisions and a + #: reader must be able to tell them apart. ``FLEET`` on every pod. + posture: CompilePosture = compile_posture.FLEET @property def underwidth(self) -> int: @@ -510,6 +517,7 @@ def facts(self) -> Dict[str, Any]: "serve_goal": bool(self.serve_goal), "mint_goal": bool(self.mint_goal), "width_reason": self.reason, + **self.posture.facts(), } for block in (self.cpu, self.memory, self.device): if block is not None: @@ -716,6 +724,7 @@ def entry_workers( limit: int = 0, device_lock: Optional[bool] = None, goals: Optional[WorkerGoals] = None, + posture: Optional[CompilePosture] = None, ) -> PoolWidth: """How many entries this pod may compile at once. @@ -746,10 +755,22 @@ def entry_workers( the constraint that actually bound. K is the mint's only multiplicative lever — two mints of one cell differed 5-vs-3 with nothing recorded to say why — so an unexplained K is a defect in itself. + + §4.30 / pgw#1137: two of the three bounds are ALSO posture-aware. The + ``goals`` above answer *"is there a tenant"*; ``posture`` answers *"is + there a human"*, and on a user's own desktop the CPU budget is halved and + the host-RAM reserve is doubled (:mod:`gen_worker.compile_posture` holds + the derivation for both). The DEVICE bound is untouched — a desktop GPU is + not shared with anyone the way its CPU and RAM are, and narrowing a bound + that already refuses to guess would only make first mints serial. The + default posture is ``FLEET``, so a pod's width is byte-identical to what it + was before this parameter existed. """ entries = max(0, int(entries)) if goals is None: goals = worker_goals.current() + if posture is None: + posture = compile_posture.current() # pgw#930 (§1.17): THREE of this policy's terms are tenant reserves, and a # pod with no SERVE goal has no tenant. `SERVING_HEADROOM_CPUS` keeps cores # for an eager forward and a heartbeat; `DEVICE_RESERVE_BYTES` keeps VRAM @@ -770,7 +791,8 @@ def entry_workers( reserve = goals.tenant_reserve_applies() cpu_headroom = SERVING_HEADROOM_CPUS if reserve else 0 device_reserve = DEVICE_RESERVE_BYTES if reserve else 0 - rss_reserve = ENTRY_RSS_RESERVE_BYTES if reserve else FORGE_RSS_RESERVE_BYTES + rss_reserve = posture.rss_reserve_bytes( + ENTRY_RSS_RESERVE_BYTES if reserve else FORGE_RSS_RESERVE_BYTES) locked = aot_device_lock.supported() if device_lock is None \ else bool(device_lock) if entries <= 1: @@ -787,7 +809,7 @@ def entry_workers( per_entry_rss_basis="not-read", per_entry_device_basis="not-read", device_lock=locked, binding="entries", ceiling=1, limit=max(0, int(limit)), - serve_goal=goals.serve, mint_goal=goals.mint, + serve_goal=goals.serve, mint_goal=goals.mint, posture=posture, reason=( f"{entries} entr{'y' if entries == 1 else 'ies'}: serial " f"(no cpu/memory/device bound was read — the entry count " @@ -798,7 +820,7 @@ def entry_workers( else: cpu = cpu_facts() vcpus = cpu.vcpus - budget = vcpus - cpu_headroom + budget = posture.cpu_budget_cores(vcpus, headroom=cpu_headroom) cpu_workers = max(1, budget // CPUS_PER_ENTRY_WORKER) if available_bytes >= 0: @@ -853,8 +875,11 @@ def entry_workers( # A caller cap NARROWS. `limit` above MAX_ENTRY_WORKERS is a caller # asking for more than the ceiling allows, and the ceiling wins. - ceiling = min(MAX_ENTRY_WORKERS, int(limit)) if limit > 0 \ - else MAX_ENTRY_WORKERS + # ...and so does the POSTURE (§4.30): a user's machine caps at half the + # fleet ceiling, and a caller cap below that still wins. Both narrow; + # neither can widen. + ceiling = posture.entry_ceiling( + min(MAX_ENTRY_WORKERS, int(limit)) if limit > 0 else MAX_ENTRY_WORKERS) # pgw#877: ONE definition of each basis. It was written twice — once for # the row, once for the reason string — which is how the reason could # have drifted from the field it explains. @@ -884,7 +909,7 @@ def _width( per_entry_device_basis=device_basis, per_entry_rss_basis=rss_basis, limit=max(0, int(limit)), - serve_goal=goals.serve, mint_goal=goals.mint) + serve_goal=goals.serve, mint_goal=goals.mint, posture=posture) workers = max( 1, min(cpu_workers, mem_workers, device_workers, ceiling, entries)) @@ -900,8 +925,13 @@ def _width( (cpu_workers, "cpu"), (mem_workers, "host-memory"), (device_workers, "vram"), (ceiling, "ceiling"), (entries, "entries"))[1] + polite = ( + " [§4.30 user-machine: half the cores, " + f"{USER_MACHINE_RSS_RESERVE_BYTES // 1024**3} GiB RAM left alone, " + f"ceiling {ceiling}, nice {posture.nice_level()}]" + if posture.user_machine else "") reason = ( - f"K={workers} ({binding}-bound, goals=" + f"K={workers} ({binding}-bound{polite}, goals=" f"{'serve+mint' if goals.serve and goals.mint else 'serve' if goals.serve else 'mint' if goals.mint else 'none'}): " f"{vcpus} vCPU ({cpu.basis}) -> " f"{cpu_workers}, {avail / 1024**3:.1f} GiB RAM ({memory.basis}) -> " @@ -2213,6 +2243,10 @@ def _rewiden(self, *, trigger: str = "measured") -> None: # not whatever a later `install()` published. goals=WorkerGoals( serve=base.serve_goal, mint=base.mint_goal), + # ...and the same rule for the posture: a mid-mint widen must + # not quietly stop being polite on a machine that was polite + # when the pool was built. + posture=base.posture, ) except Exception: # noqa: BLE001 — a re-derivation never fails a mint logger.debug("aot-pool: width re-derivation failed", exc_info=True) diff --git a/src/gen_worker/compile_posture.py b/src/gen_worker/compile_posture.py new file mode 100644 index 00000000..d418e616 --- /dev/null +++ b/src/gen_worker/compile_posture.py @@ -0,0 +1,225 @@ +"""WHOSE machine this mint is running on — declared by the entry point. + +DESIGN-RULINGS §4.30 (Paul, 2026-08-11), verbatim: *"Be nice, especially on +cozy-local — never saturate the user's only machine. Compile parallelism K is +POSTURE-AWARE: aggressive on a dedicated serving pod, gentle/niced/capped on +cozy-local, accepting a slower local mint to stay polite."* + +Every clamp in :mod:`gen_worker.aot_compile_pool` is written for a SERVING POD +with a co-resident tenant: ``SERVING_HEADROOM_CPUS`` keeps two cores for an +asyncio beat and an eager forward, ``ENTRY_RSS_RESERVE_BYTES`` keeps host RAM +so a tenant request does not meet the OOM killer. Those are the right terms for +a datacenter box we bought. They are the wrong terms for a desktop, where the +co-resident workload is a human — who outranks the mint, cannot be scheduled +around, and will simply stop using the product if their machine stops +responding. + +THE FACT, AND WHY IT IS DECLARED RATHER THAN SNIFFED +---------------------------------------------------- +The fact is *"a person is sitting at this machine"*, and no process can measure +it. Three plausible-looking proxies exist in this tree and every one of them is +a DIFFERENT question: + +* ``local_cell_store.trust_class() == "untrusted"`` — the HUB's verdict on + whether this hardware may publish a cell. A rented community-cloud pod is + untrusted and has no human on it; being polite there would slow work we are + paying for by the second. Trust is not tenancy. +* ``publisher is None`` at ``local_serve``'s call site — a fact about the SINK. + ``publish_disarmed`` (a pgw#980 probe) makes an ordinary fleet pod look the + same, and that pod should compile flat out. +* ``worker_goals`` — what the pod was BOUGHT to do. cozy-local was not bought. + +pgw#1127 §2 named the failure mode this codebase keeps hitting: *"two +derivations of one fact"*. So this is not derived at all. It is DECLARED, once, +by the process entry that knows: :mod:`gen_worker.local_serve` — the cozy-local +CLI's only arming entry — passes :data:`USER_MACHINE`, and everything else gets +:data:`FLEET` by construction because that is the struct's default. + +And it is deliberately not an environment variable. §1.17, verbatim: *"Envs are +for secrets and configuration, not for logic gates."* Politeness changes what +the machine DOES, so it travels as a typed value on the parent->child +``MintRequest`` — the same wire that already carries ``vram_cap_bytes`` and +``device`` — never as an ambient toggle a stray shell export could flip. + +WHAT POLITENESS MEANS, AND WHY EACH TERM +----------------------------------------- +The policy lives here, as methods, so no caller ever branches on the boolean +and the four terms cannot drift apart: + +* :meth:`nice_level` — **the primary CPU lever, and the only one that actually + preserves interactivity.** Reserving cores does not: the scheduler will still + put a compile on the core the compositor wants. Priority says who yields, at + microsecond granularity, in the kernel. It is also strictly better than a + reservation in the other direction — a niced mint uses the WHOLE machine + while the user is away and gets out of the way the moment they come back, so + politeness costs nothing when nobody is looking. +* :meth:`cpu_budget_cores` — nice does nothing for the things that are not CPU + time. K children mean K concurrent ``cc1plus``, K inductor caches being + written, and K working sets in the page cache; that pressure is felt as + stutter whatever the priority. ``CPUS_PER_ENTRY_WORKER`` is an AVERAGE (one + core for ~71 % of an entry, up to ``compile_threads`` for the rest), so a pod + deliberately overcommits: at the burst it asks for ~2x the box, which is + correct when the box is ours and otherwise idle. Halving the budget makes the + BURST ask for the machine instead of double it. +* :meth:`entry_ceiling` — the disk and the page cache being contended are the + user's own. Half the fleet ceiling. +* :meth:`rss_reserve_bytes` — **a mint that OOMs a desktop is worse than a slow + one.** ``MemAvailable`` counts reclaimable page cache as free, so a pool sized + against it will evict the working set of every application the user has open + before it meets any limit. The reserve is what stops that, and on a desktop + the OOM killer's most attractive target is a browser with forty tabs. + +WHAT IS DELIBERATELY NOT HERE +------------------------------ +**No interactivity probe.** "Pause the mint while the user is typing" was +considered and rejected: it is a cross-platform mess (X11 idle time, Wayland +has no equivalent, macOS has a third), and it is wrong in the case that matters +— a developer running a build in a terminal produces no input events at all and +is exactly the person who needs the CPU. :meth:`nice_level` already IS the +yield, implemented by the scheduler, for free. + +**No ionice.** It would be the right instinct — priority for I/O the way nice +is for CPU — but ``ioprio_set`` has no stdlib wrapper (a ctypes syscall), and +on the ``none``/``mq-deadline`` schedulers that ship on modern desktops it is a +no-op. The RAM reserve and the entry ceiling bound the I/O instead, by bounding +the number of concurrent writers. Revisit only with a measurement showing disk +contention, not on instinct. + +**No third posture.** A community-cloud pod is untrusted hardware that is still +rented by the second with nobody sitting at it; it compiles as FLEET, and that +is not an oversight. +""" + +from __future__ import annotations + +from typing import Optional + +import msgspec + +#: The lowest scheduling priority a process may ask for. Not a tunable: there +#: is no case for "somewhat polite" on a machine whose owner is using it, and +#: a niced compile still runs at full speed on an otherwise idle box. +USER_MACHINE_NICE = 19 + +#: The share of the pod's core budget a mint may take on a user's machine. +#: See :meth:`CompilePosture.cpu_budget_cores` for the derivation. +USER_MACHINE_CPU_SHARE = 2 + +#: Entry children a user's machine may run at once, against +#: ``aot_compile_pool.MAX_ENTRY_WORKERS`` (8) on a pod. Half, because the disk +#: whose write amplification that constant bounds is the user's own — and +#: because K only buys whole ROUNDS, so the difference between 4 and 8 is one +#: extra round on the largest real family, paid in wall-clock a desktop mint +#: is already trading away. +USER_MACHINE_MAX_ENTRY_WORKERS = 4 + +#: Host RAM a user's machine keeps for its owner, against +#: ``aot_compile_pool.ENTRY_RSS_RESERVE_BYTES`` (4 GiB) on a pod. Sized as one +#: browser plus one editor plus the page cache they are both living in — the +#: things whose eviction IS the experience of a machine "getting slow", and +#: which ``MemAvailable`` cheerfully reports as free. +USER_MACHINE_RSS_RESERVE_BYTES = 8 * 1024**3 + + +class CompilePosture(msgspec.Struct, frozen=True, kw_only=True): + """Whose machine a mint runs on. Passed, never inferred. + + Travels on ``mint_process.MintRequest`` so the process that sizes the + compile pool — the mint child, three frames below the entry that knows — is + told rather than left to guess. + """ + + #: True when the machine running this mint belongs to the person using the + #: product, and they are on it. cozy-local, and nothing else today. + user_machine: bool = False + + # -- the policy, so nothing else branches on the flag ------------------ + + def nice_level(self) -> int: + """Scheduling-priority increment for the mint process tree. + + Applied by the mint child TO ITSELF and inherited by every descendant + — the entry-pool children, inductor's compile workers, and every + ``cc1plus`` under them. 0 on a pod: a mint there competes with a tenant + whose reserves are already held back explicitly, and de-prioritising it + on top of that would slow paid work for no gain. + """ + return USER_MACHINE_NICE if self.user_machine else 0 + + def cpu_budget_cores(self, vcpus: int, *, headroom: int) -> int: + """Cores the pool may size itself against. + + A pod takes everything but the tenant's ``headroom``. A user's machine + takes the NARROWER of that and half the box, so that + ``K * compile_threads`` at the burst asks for the machine rather than + double it. The two bounds compose the way a reader expects at both + ends: on a 4-core laptop the headroom binds (4-2 = 2 -> K=1, serial); + on a 32-core workstation the half binds (16, not 30). + """ + budget = int(vcpus) - int(headroom) + if self.user_machine: + budget = min(budget, int(vcpus) // USER_MACHINE_CPU_SHARE) + return budget + + def entry_ceiling(self, default: int) -> int: + """The hard cap on concurrent entry children. Never widens.""" + if self.user_machine: + return min(int(default), USER_MACHINE_MAX_ENTRY_WORKERS) + return int(default) + + def rss_reserve_bytes(self, default: int) -> int: + """Host RAM the pool must leave alone. Never shrinks.""" + if self.user_machine: + return max(int(default), USER_MACHINE_RSS_RESERVE_BYTES) + return int(default) + + def facts(self) -> dict: + """The posture as it rides the width row, so a K nobody expected can + be explained without re-deriving anything.""" + return { + "posture": "user-machine" if self.user_machine else "fleet", + "nice": self.nice_level(), + } + + +#: A pod we bought: no human on it, compile flat out. The default everywhere, +#: which is what keeps the serving path unchanged by construction. +FLEET = CompilePosture(user_machine=False) + +#: A machine whose owner is sitting at it. Declared by ``local_serve`` only. +USER_MACHINE = CompilePosture(user_machine=True) + + +_INSTALLED: Optional[CompilePosture] = None + + +def install(posture: CompilePosture) -> None: + """Publish this process's posture. + + One carrier, one moment (§4.22), exactly like ``worker_goals.install``: + the mint child publishes what its request declared, before anything sizes + a pool. A test installs a value; it does not clear a cache. + """ + global _INSTALLED + _INSTALLED = posture + + +def current() -> CompilePosture: + """The installed posture, or :data:`FLEET`. + + The fallback is the right reading for a library import and for every + serving pod, and it is why nothing on the fleet path had to change. + """ + return FLEET if _INSTALLED is None else _INSTALLED + + +__all__ = [ + "USER_MACHINE_MAX_ENTRY_WORKERS", + "USER_MACHINE_NICE", + "USER_MACHINE_RSS_RESERVE_BYTES", + "CompilePosture", + "FLEET", + "USER_MACHINE", + "current", + "install", +] diff --git a/src/gen_worker/local_serve.py b/src/gen_worker/local_serve.py index 5cd0e3cb..7e3ceedc 100644 --- a/src/gen_worker/local_serve.py +++ b/src/gen_worker/local_serve.py @@ -38,17 +38,25 @@ import asyncio import logging +import sys +import time from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, Mapping, Optional, Tuple from . import activity as activity_mod +from . import aot_compile_pool from . import compile_cache as cc -from . import fleet_cells, mint_delegate -from .mint_process import MintSlot +from . import compile_posture, fleet_cells, local_cell_store, mint_delegate +from .mint_process import MintFrame, MintSlot logger = logging.getLogger(__name__) +#: Seconds between progress lines while a compile runs. A mint is minutes-to- +#: an-hour of work, so a line a second would be noise and a line a minute +#: would read as a hang. Ten is the same cadence the fleet's own beat uses. +NOTICE_INTERVAL_S = 10.0 + @dataclass(frozen=True) class LocalMintContext: @@ -123,6 +131,95 @@ def enable_compiled( return _mint_here(pipe, pending, mint) +def _say(line: str) -> None: + """One line to the person at the keyboard. + + ``print`` to stderr and not ``logger``: the CLI configures logging for the + endpoint's own output, a mint runs before any of that matters, and every + existing progress signal on this path already goes somewhere a user does + not look (an activity addressed to a hub that cozy-local does not have, or + a ``logger.info`` inside a subprocess). + """ + print(f"cozy: {line}", file=sys.stderr, flush=True) + + +def compile_notice( + family: str, posture: compile_posture.CompilePosture, + *, cpu: Optional[aot_compile_pool.CpuFacts] = None, + store_root: Optional[Path] = None, +) -> str: + """What a user is told BEFORE a compile starts — §4.30's honesty half. + + It has to answer four questions a support ticket would otherwise ask: + *what is happening*, *will it happen again*, *what is it costing me right + now*, and *may I stop it*. The cost figures are this machine's own + (``cpu_facts`` reads the same cgroup/affinity/host triple the pool does), + and they are stated as a CEILING because the mint child narrows further on + memory and VRAM — overstating what we took would be the same defect as + saying nothing. + """ + facts = cpu if cpu is not None else aot_compile_pool.cpu_facts() + vcpus = max(1, int(facts.vcpus)) + cores = max( + 1, posture.cpu_budget_cores( + vcpus, headroom=aot_compile_pool.SERVING_HEADROOM_CPUS)) + workers = posture.entry_ceiling(aot_compile_pool.MAX_ENTRY_WORKERS) + root = store_root if store_root is not None else local_cell_store.store_root() + reserve = posture.rss_reserve_bytes( + aot_compile_pool.ENTRY_RSS_RESERVE_BYTES) // 1024**3 + return ( + f"compiling {family} for this machine — this happens ONCE; every " + f"later run of this endpoint arms from {root} with no compile and no " + f"network. It takes a while.\n" + f" This is your machine, so the compile is polite: lowest CPU " + f"priority (nice {posture.nice_level()}), at most {workers} parallel " + f"worker(s) sized against {cores} of your {vcpus} cores, and " + f"{reserve} GiB of RAM left alone.\n" + f" Ctrl-C is safe — finished work is kept and the next run " + f"picks up where this one stopped." + ) + + +class _Progress: + """Renders the child's frames onto the user's terminal, throttled. + + Deliberately a throttle on TIME and not on frame count: the frames arrive + at wildly different rates (one per export, one per compiled entry, then + silence through a single long link step), and a user watching a machine + they can no longer type on needs a steady "still working" cadence, not a + burst followed by nothing. + """ + + def __init__( + self, family: str, *, say: Any = _say, + interval_s: float = NOTICE_INTERVAL_S, clock: Any = time.monotonic, + ) -> None: + self.family = family + self.say = say + self.interval_s = float(interval_s) + self.clock = clock + self.started = clock() + self._last: Optional[float] = None + self.lines = 0 + + def elapsed(self) -> str: + secs = int(max(0.0, self.clock() - self.started)) + return f"{secs // 60}m{secs % 60:02d}s" + + def __call__(self, frame: MintFrame) -> None: + now = self.clock() + if self._last is not None and now - self._last < self.interval_s: + return + self._last = now + step = ( + f" {frame.step}/{frame.total}" + if frame.total > 0 else "") + self.say( + f"[{self.family}] {frame.phase or 'compiling'}{step} — " + f"{self.elapsed()} elapsed") + self.lines += 1 + + def _mint_here( pipe: Any, pending: "fleet_cells.PendingSelfMint", mint: LocalMintContext, ) -> bool: @@ -130,12 +227,22 @@ def _mint_here( Identical to the executor's ``_delegated_mint_run`` in everything that decides correctness — same ``build_cell``, same child, same - ``adopt_delegated_mint`` gate — and different in the two things a desktop + ``adopt_delegated_mint`` gate — and different in the three things a desktop does not have: there is no serving loop to keep beating (this call is the - boot, and it blocks), and there is nowhere to publish, so the obligation - ends at ``keep_self_mint_local`` instead of at the publish gate. + boot, and it blocks), there is nowhere to publish, so the obligation ends + at ``keep_self_mint_local`` instead of at the publish gate, and there is a + PERSON on the machine — so §4.30's posture is declared here, and what the + compile is doing to their computer is said out loud (pgw#1137). """ result: Optional[mint_delegate.DelegatedResult] = None + family = str(pending.family) + # §4.30 / pgw#1137: the ONE site in the tree that declares a user-machine + # posture. It is stated here, not derived from `publisher is None` or from + # `local_cell_store.trust_class()` — both are facts about the SINK, and a + # community-cloud pod matches them while having no human on it. + posture = compile_posture.USER_MACHINE + _say(compile_notice(family, posture)) + watch = _Progress(family) with activity_mod.running(activity_mod.KIND_SELF_MINT_COMPILE) as act: try: result = _drive(mint_delegate.build_cell( @@ -150,12 +257,22 @@ def _mint_here( # its next boot is what this mint stamps (pgw#686). weight_lane=cc.cell_base_execution_lane(pipe), device=mint.device, + posture=posture, ), - act=act)) + act=act, watch=watch)) except Exception as exc: # noqa: BLE001 — a mint must never kill a serve logger.warning( "local-serve: the mint for %s failed (%s: %s); serving eager " "and keeping nothing", pending.family, type(exc).__name__, exc) + if result is not None and result.ok: + _say( + f"compiled {family} in {watch.elapsed()} — kept at " + f"{local_cell_store.store_root()}; later runs arm from it.") + else: + _say( + f"{family} is not compiled on this machine ({watch.elapsed()} " + f"spent); serving eager. Finished compile work, if any, is kept " + f"for the next run.") if result is not None and result.ok: # The cell is ALREADY in this machine's store — `adopt_delegated_mint` # put it there before any publish could be attempted, which is what @@ -239,5 +356,6 @@ def slot_map( __all__ = [ - "LocalMintContext", "enable_compiled", "mint_context", "slot_map", + "LocalMintContext", "compile_notice", "enable_compiled", "mint_context", + "slot_map", ] diff --git a/src/gen_worker/mint_child.py b/src/gen_worker/mint_child.py index 08898b51..aeda6354 100644 --- a/src/gen_worker/mint_child.py +++ b/src/gen_worker/mint_child.py @@ -69,7 +69,7 @@ import msgspec -from . import warm_spans, worker_goals +from . import compile_posture, warm_spans, worker_goals from .api.errors import ValidationError from .api.export_contract import ( blocker_refusal, export_declaration, open_blockers) @@ -1182,6 +1182,69 @@ def _install_goals() -> None: goals.serve, goals.mint, goals.declared, goals.declaration_understood) +def _install_posture(request: MintRequest) -> None: + """Adopt the posture the parent DECLARED, and act on it — §4.30. + + Three things happen here and they all belong at this one point, before a + single graph is exported: + + **1. Publish it**, so ``aot_compile_pool.entry_workers`` — three frames + down, in this process — sizes K against it. Same carrier and same single + moment as :func:`_install_goals`. + + **2. Drop this process's scheduling priority**, which is the whole of the + CPU half of politeness. It is done HERE, by the child to itself, and NOT + with a ``preexec_fn`` on the spawn, for the reason + ``aot_compile_pool.arm_parent_death_signal`` already writes down: a + ``preexec_fn`` forces ``fork()`` instead of ``posix_spawn()`` for a process + that has live gRPC threads with ``pthread_atfork`` handlers, which is a + large blast radius for a one-line guarantee. + + Doing it here also covers strictly more ground than nicing the entry-pool + spawns would. Since pgw#1080 every production mint is weight-free, and + ``aot_mint._mint_cell`` forces ``parallel=False`` for a weight-free mint — + so the pool spawns NO children at all today and the compile runs in THIS + process. Nice is inherited across ``fork``/``exec``, so this one call + covers the serial compile, the entry children when parallelism returns + (pgw#1111), inductor's own compile workers, and every ``cc1plus`` under + them. + + **3. Arm ``PR_SET_PDEATHSIG``** on a user's machine. ``mint_process`` spawns + this child with ``start_new_session=True``, so it holds its OWN session and + a terminal's Ctrl-C or SIGHUP never reaches it. On a pod that is correct — + the parent reaps the group deliberately when it abandons a mint. On a + desktop the "parent" is a CLI that a human closes, and without this a + closed terminal leaves a full-speed compile tree running with nobody left + to reap it: the exact "my machine is at a crawl and I don't know why" + support ticket politeness exists to prevent. Nothing is lost by dying — + ``MintRequest.resume`` points at ``aot_resume``'s cross-attempt bank, which + lives outside the per-attempt workdir precisely so finished entries survive + into the next run. + + Never fatal. A kernel that refuses either call leaves a mint that is rude + rather than no mint at all, and says so. + """ + posture = request.posture + compile_posture.install(posture) + if not posture.user_machine: + return + from .aot_compile_pool import arm_parent_death_signal + + level = posture.nice_level() + applied = -1 + try: + applied = os.nice(level) + except (OSError, AttributeError): + logger.warning( + "mint-child: could not lower this mint's scheduling priority " + "(nice %d) — it will compile at ordinary priority on a machine " + "someone is using", level, exc_info=True) + reaped = arm_parent_death_signal() + logger.info( + "mint-child: §4.30 user-machine posture — nice %d (now %d), " + "dies-with-parent=%s", level, applied, reaped) + + def main(argv: Optional[Sequence[str]] = None) -> int: args = list(sys.argv[1:] if argv is None else argv) logging.basicConfig( @@ -1200,6 +1263,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: return EXIT_BAD_REQUEST _install_goals() + _install_posture(request) report_path = Path(request.report) started = time.monotonic() try: diff --git a/src/gen_worker/mint_delegate.py b/src/gen_worker/mint_delegate.py index 81a6eed0..41a18b6f 100644 --- a/src/gen_worker/mint_delegate.py +++ b/src/gen_worker/mint_delegate.py @@ -33,11 +33,12 @@ import tempfile from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, Mapping, Optional, Tuple +from typing import Any, Callable, Dict, Mapping, Optional, Tuple from . import activity as activity_mod from . import aot_resume from . import boot_phases +from . import compile_posture from . import mint_budget from . import mint_process from . import progress as progress_mod @@ -89,6 +90,11 @@ class MintTask: execution_lane: str = "" configs: Dict[str, Dict[str, Any]] = field(default_factory=dict) device: Optional[int] = None + #: §4.30 / pgw#1137: whose machine this mint will run on. DECLARED here by + #: the caller rather than read off a process global, so the one entry that + #: knows (``local_serve``) states it and every other caller gets ``FLEET`` + #: by construction — there is no ambient value to forget to set. + posture: compile_posture.CompilePosture = compile_posture.FLEET @dataclass(frozen=True) @@ -316,6 +322,7 @@ def build_request( vram_cap_bytes=int(cap_bytes), execution_lane=task.execution_lane, configs={k: dict(v) for k, v in task.configs.items()}, + posture=task.posture, ) @@ -335,7 +342,7 @@ def _pool_stat(phases: Any, key: str) -> int: return 0 -def _on_frame(act: Any) -> Any: +def _on_frame(act: Any, watch: Optional[Watcher] = None) -> Any: def _apply(frame: mint_process.MintFrame) -> None: # No new protocol: the child's phase lands on the SAME # self_mint_compile activity the hub already reads, and ships on the @@ -345,6 +352,12 @@ def _apply(frame: mint_process.MintFrame) -> None: act.phase(frame.phase, frame.step, frame.total) if frame.note: act.note(frame.note[:200]) + # pgw#1137: ...and, on a machine with a person at it, onto that + # person's terminal. The activity above is addressed to the HUB, and + # cozy-local has no hub — so on a desktop every frame of a 20-minute + # compile went nowhere a user could see it. + if watch is not None: + watch(frame) return _apply @@ -382,12 +395,20 @@ def _apply(value: float) -> None: return _apply +#: pgw#1137: a sink for the child's progress frames that is NOT the hub. The +#: fleet passes nothing and behaves exactly as before; ``local_serve`` passes a +#: terminal renderer, because a 20-minute compile a user cannot see is a +#: support ticket regardless of how correct it is. +Watcher = Callable[[mint_process.MintFrame], None] + + async def build_cell( task: MintTask, *, act: Any, abandon: Any = None, max_attempts: int = mint_process.MAX_ATTEMPTS, + watch: Optional[Watcher] = None, ) -> DelegatedResult: """Build and adopt one cell in a child process. Never raises for a mint failure — the worker must never die with its mint.""" @@ -444,7 +465,7 @@ async def build_cell( family, task.weight_lane)) act.phase(activity_mod.PHASE_LOAD) outcome = await mint_process.run_mint( - request, workdir=workdir, on_frame=_on_frame(act), + request, workdir=workdir, on_frame=_on_frame(act, watch), on_evidence=_on_evidence(act), abandon=abandon) last = outcome.detail diff --git a/src/gen_worker/mint_process.py b/src/gen_worker/mint_process.py index 4889c20d..de4aff1c 100644 --- a/src/gen_worker/mint_process.py +++ b/src/gen_worker/mint_process.py @@ -70,8 +70,9 @@ import msgspec -from . import cell_key +from . import cell_key, compile_posture from .api.binding import ModelRef, binding_wire_refs, wire_ref +from .compile_posture import CompilePosture from .stall import SilenceWindow logger = logging.getLogger(__name__) @@ -297,6 +298,16 @@ class MintRequest(msgspec.Struct, frozen=True, kw_only=True): #: different config traces different graphs and the parent's proof misses. execution_lane: str = "" configs: Dict[str, Dict[str, Any]] = {} + #: §4.30 / pgw#1137: WHOSE MACHINE this mint runs on, declared by the + #: process entry that knows (``local_serve`` says user-machine; nothing + #: else says anything, so every fleet mint gets the ``FLEET`` default). + #: + #: It has to travel on the request for the same reason ``vram_cap_bytes`` + #: does — the pool's width is computed INSIDE this child — and it travels + #: as a typed value rather than an env because §1.17 says an env may carry + #: a VALUE and may not carry a DECISION. Politeness is a decision: it + #: changes the nice level of every process in the mint tree and halves K. + posture: CompilePosture = compile_posture.FLEET class MintFrame(msgspec.Struct, frozen=True, kw_only=True): diff --git a/tests/test_compile_politeness_pgw1137.py b/tests/test_compile_politeness_pgw1137.py new file mode 100644 index 00000000..f01379d1 --- /dev/null +++ b/tests/test_compile_politeness_pgw1137.py @@ -0,0 +1,726 @@ +"""pgw#1137 / DESIGN-RULINGS §4.30 — a mint on the USER'S OWN MACHINE is polite. + +pgw#1127 §7 item 6 filed this and left it unowned: *"posture-aware K, niced +children, never saturate the user's only machine… before any cozy-local +release."* Its own §6 states the evidence — **zero ``os.nice`` calls anywhere in +``src/``**, and every clamp in ``aot_compile_pool`` written for a serving pod +with a co-resident TENANT rather than for a desktop with a co-resident HUMAN. + +Since pgw#1127 S1 (`b29a5e95`) ``cozy serve`` drives the ordinary +``mint_delegate`` -> ``mint_process`` -> ``mint_child`` chain, so that saturation +is now a shipping product behaviour and not a hypothetical. + +WHAT IS PROVEN HERE, and what each section would have looked like before + +1. **The posture is DECLARED.** ``local_serve`` — the one entry that knows — + states it; it is not sniffed off ``trust_class()`` or off ``publisher is + None`` (both are facts about the SINK, and a rented community-cloud pod + matches them while having nobody sitting at it), and it is not an env var + (§1.17: an env may carry a VALUE, not a DECISION). RED: ``MintTask`` and + ``MintRequest`` had no posture field and ``compile_posture`` did not exist. +2. **CPU.** The pool halves its core budget and the mint child drops its own + scheduling priority — by the CHILD, on itself, never through a + ``preexec_fn``. RED: ``entry_workers`` had no posture parameter and nothing + in ``src/`` called ``os.nice``. +3. **Memory.** More host RAM is left alone, because ``MemAvailable`` counts the + user's page cache as free and a mint that OOMs a desktop is worse than a + slow one. RED: one reserve constant, sized for a pod. +4. **Pods do not regress.** The default posture is ``FLEET`` and a pod's width + is arithmetically identical to what it was — proven against an independent + re-implementation of the pre-issue formula, not against the code under test. +5. **Honesty.** The user can see the compile and what it is costing them. RED: + the only progress signal on this path was an activity addressed to a hub + cozy-local does not have, plus ``logger.info`` inside a subprocess. +6. **Interruptibility.** The mint child dies with its parent, and what it + finished survives into the next run. RED: the child holds its own session + (``start_new_session=True``), so a closed terminal left a full-speed compile + tree running with nobody to reap it. +""" + +from __future__ import annotations + +import ast +import os +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Tuple + +import msgspec +import pytest + +from gen_worker import ( + aot_compile_pool, compile_posture, local_serve, mint_child, mint_delegate, + mint_process) +from gen_worker.compile_posture import FLEET, USER_MACHINE, CompilePosture +from gen_worker.mint_process import MintFrame, MintRequest, MintSlot +from gen_worker.worker_goals import SERVE_ONLY + +SRC = Path(aot_compile_pool.__file__).parent + + +@pytest.fixture(autouse=True) +def _no_installed_posture() -> Iterator[None]: + """The posture is a per-process publication; a test installs a value and + every other test must not inherit it.""" + compile_posture.install(FLEET) + yield + compile_posture.install(FLEET) + + +def _width( + entries: int, *, vcpus: int, avail_gib: float, posture: CompilePosture, + free_vram_gib: float = 0.0, device_gib: float = 0.0, limit: int = 0, +) -> aot_compile_pool.PoolWidth: + """``entry_workers`` driven off SUPPLIED facts only. + + Every reading the function would otherwise probe (cores, cgroup, card) is + passed in, so these cases describe a laptop and a workstation from a + 32-core CI-less box without simulating anything the policy decides. + """ + return aot_compile_pool.entry_workers( + entries, vcpus=vcpus, + available_bytes=int(avail_gib * 1024**3), + free_vram_bytes=int(free_vram_gib * 1024**3), + device_bytes=int(device_gib * 1024**3), + device_lock=True, goals=SERVE_ONLY, posture=posture, limit=limit) + + +# --------------------------------------------------------------------------- +# 1. The posture is DECLARED — not sniffed off the sink, not an env var +# --------------------------------------------------------------------------- + + +class _Task: + """Captures the ``MintTask`` ``local_serve`` builds, without minting.""" + + def __init__(self) -> None: + self.task: Any = None + self.watch: Any = None + + def build_cell(self, task: Any, **kw: Any) -> Any: + self.task = task + self.watch = kw.get("watch") + + class _Coro: + def close(self) -> None: + pass + + def __await__(self) -> Any: # pragma: no cover - never awaited here + yield + return None + + return _Coro() + + +class _Cfg: + shapes: Tuple[Tuple[int, int], ...] = ((64, 64),) + targets: Tuple[str, ...] = ("transformer",) + family = "micro-diffusion" + lora_bucket = 0 + guidance_scales: Tuple[float, ...] = () + text_lens: Tuple[int, ...] = () + + +class _Pending: + family = "micro-diffusion" + arm_token = "arm2-" + "1" * 56 + mint_root = "/tmp/pgw1137-does-not-exist" + cfg = _Cfg() + + +def test_the_local_serve_entry_DECLARES_the_user_machine_posture( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """§4.30's input, stated at the one site that knows. + + RED before this issue: ``MintTask`` had no ``posture`` field at all, so the + mint child sized its pool from a fleet policy on a desktop. + """ + cap = _Task() + monkeypatch.setattr(mint_delegate, "build_cell", cap.build_cell) + monkeypatch.setattr(local_serve, "_drive", lambda coro: None) + monkeypatch.setattr( + local_serve.cc, "cell_base_execution_lane", lambda pipe: "bf16") + monkeypatch.setattr(local_serve, "_say", lambda line: None) + monkeypatch.setattr( + local_serve.fleet_cells, "terminus_of", lambda p: "already-ended") + + ctx = local_serve.mint_context( + function="generate", module="micro_diffusion.endpoint", + slots={"pipeline": MintSlot(ref="cozy/micro#1", path="/tmp/micro")}) + local_serve._mint_here(object(), _Pending(), ctx) # type: ignore[arg-type] + + assert cap.task is not None, "the local path must build a MintTask" + assert cap.task.posture == USER_MACHINE, ( + "cozy-local runs on the machine the user is sitting at; the posture " + "that sizes and nices the mint has to be DECLARED here, because " + "nothing downstream can measure whether a person is present") + + +def test_the_posture_is_not_derived_from_the_sink_or_from_the_trust_class( +) -> None: + """The three near-miss proxies, fenced. + + pgw#1127 §2's own warning is *"two derivations of one fact"*. Trust + (``local_cell_store.trust_class()``) answers whether the HUB will accept a + cell from this hardware; ``publisher is None`` answers whether a sink was + wired. A community-cloud pod satisfies both and is rented by the second + with nobody on it — being polite there would slow work we are paying for. + """ + tree = ast.parse((SRC / "compile_posture.py").read_text()) + # Identifiers the CODE touches. Read off the AST and not off the text, so + # the module may (and does) explain in prose why each proxy was rejected + # without the fence mistaking the explanation for a dependency. + used = { + node.id for node in ast.walk(tree) if isinstance(node, ast.Name) + } | { + node.attr for node in ast.walk(tree) if isinstance(node, ast.Attribute) + } | set(_module_names(SRC / "compile_posture.py")) + for proxy in ("trust_class", "local_cell_store", "CellPublisher", + "worker_goals", "publisher", "fleet_cells"): + assert proxy not in used, ( + f"compile_posture must not derive the posture from {proxy!r} — " + f"it is a different question with a different authority") + + +def test_no_environment_variable_can_switch_politeness() -> None: + """§1.17, verbatim: *"Envs are for secrets and configuration, not for logic + gates."* Politeness changes the nice level of a whole process tree and + halves K, so it is a decision and it travels as a typed value. + """ + tree = ast.parse((SRC / "compile_posture.py").read_text()) + reads = [ + node for node in ast.walk(tree) + if isinstance(node, ast.Attribute) + and node.attr in ("environ", "getenv", "environb") + ] + assert not reads, ( + "the posture must not be readable from the environment: an ambient " + "toggle can be flipped by a stray shell export and cannot be reasoned " + "about from the request that produced a cell") + assert "os" not in { + alias.name + for node in ast.walk(tree) if isinstance(node, ast.Import) + for alias in node.names + }, "compile_posture holds policy, not process manipulation" + + +def test_the_posture_survives_the_parent_to_child_WIRE() -> None: + """The child sizes the pool, so the declaration has to reach it. + + It rides ``MintRequest`` — the same JSON file that already carries + ``vram_cap_bytes`` and ``device`` — so the round trip through msgspec is + the property, not the in-process object identity. + """ + task = mint_delegate.MintTask( + pending=_Pending(), pipe=None, function="generate", + modules=("micro_diffusion.endpoint",), posture=USER_MACHINE) + request = mint_delegate.build_request( + task, workdir=Path("/tmp/pgw1137"), cap_bytes=0) + assert request.posture == USER_MACHINE + + decoded = msgspec.json.decode( + msgspec.json.encode(request), type=MintRequest) + assert decoded.posture.user_machine is True, ( + "the width is computed INSIDE the mint child, whose only input is " + "this file — a posture that does not survive the encode is a posture " + "the pool never sees") + + +def test_a_fleet_mint_declares_nothing_and_gets_the_fleet_posture() -> None: + """The non-regression that makes the whole design safe: FLEET is the + struct default, so every caller that has not heard of this issue — the + executor, every scheduled mint — is unchanged by construction.""" + task = mint_delegate.MintTask( + pending=_Pending(), pipe=None, function="generate", modules=("m",)) + assert task.posture == FLEET + request = mint_delegate.build_request( + task, workdir=Path("/tmp/pgw1137"), cap_bytes=0) + assert request.posture == FLEET + assert MintRequest( + function="f", modules=(), family="x", arm_token="", target="", + work_root="", report="", + cfg=mint_process.CompileCellSpec()).posture == FLEET + + +# --------------------------------------------------------------------------- +# 2. CPU — half the cores, and the lowest priority the scheduler offers +# --------------------------------------------------------------------------- + + +def test_a_user_machine_pool_takes_at_most_HALF_the_cores() -> None: + """``CPUS_PER_ENTRY_WORKER`` is an AVERAGE (one core for ~71 % of an entry, + up to ``compile_threads`` for the rest), so a pod deliberately overcommits: + at the burst K*compile_threads asks for ~2x the box, which is right when + the box is ours and idle. Halving the budget makes the burst ask for the + machine instead of double it. + + RED: ``entry_workers`` had no posture, so a 32-core desktop sized K off + (32-2)//2 = 15 exactly as a 32-core pod did. + """ + pod = _width(36, vcpus=32, avail_gib=256, posture=FLEET) + desk = _width(36, vcpus=32, avail_gib=256, posture=USER_MACHINE) + assert pod.cpu_workers == 15 + assert desk.cpu_workers == 8, ( + "half of 32 cores, over 2 cores per entry worker — a quarter of the " + "machine's cores as a steady ask, one whole machine at the burst") + + +@pytest.mark.parametrize( + "vcpus,expected", [(2, 1), (4, 1), (8, 2), (16, 4), (32, 8)]) +def test_the_core_budget_degrades_sanely_from_a_laptop_to_a_workstation( + vcpus: int, expected: int, +) -> None: + """Both bounds compose at the ends: on a 4-core laptop the serving headroom + binds ((4-2)//2 = 1, i.e. the serial in-process path, which is the honest + answer); on a 32-core workstation the half binds.""" + assert _width( + 36, vcpus=vcpus, avail_gib=256, + posture=USER_MACHINE).cpu_workers == expected + + +def test_a_four_core_laptop_compiles_SERIALLY() -> None: + """The product statement of the row above. K=1 is the pre-pgw#809 serial + path and it is what a machine with four cores and a person on it should + do.""" + assert _width( + 36, vcpus=4, avail_gib=16, posture=USER_MACHINE).workers == 1 + + +def test_the_entry_CEILING_halves_on_a_user_machine() -> None: + """K children mean K concurrent ``cc1plus``, K inductor caches being + written and K working sets in the page cache — and here that disk and that + page cache are the user's own. RED: one ceiling, 8, for every machine.""" + fat = _width(36, vcpus=128, avail_gib=512, posture=USER_MACHINE) + assert fat.ceiling == compile_posture.USER_MACHINE_MAX_ENTRY_WORKERS == 4 + assert fat.workers == 4, ( + "a workstation big enough to ignore every other bound still stops at " + "the posture ceiling") + assert _width( + 36, vcpus=128, avail_gib=512, + posture=FLEET).ceiling == aot_compile_pool.MAX_ENTRY_WORKERS + + +def test_a_caller_cap_below_the_posture_ceiling_still_wins() -> None: + """Both narrow; neither widens. An operator forcing K=2 must not be + widened to 4 by the posture, and the posture must not be widened to 8 by + an operator asking for it.""" + assert _width( + 36, vcpus=128, avail_gib=512, posture=USER_MACHINE, + limit=2).ceiling == 2 + assert _width( + 36, vcpus=128, avail_gib=512, posture=USER_MACHINE, + limit=8).ceiling == 4 + + +class _Nice: + def __init__(self) -> None: + self.levels: List[int] = [] + + def __call__(self, inc: int) -> int: + self.levels.append(int(inc)) + return int(inc) + + +def _drive_child_posture( + monkeypatch: pytest.MonkeyPatch, posture: CompilePosture, +) -> Tuple[_Nice, List[bool]]: + nice = _Nice() + armed: List[bool] = [] + monkeypatch.setattr(os, "nice", nice) + monkeypatch.setattr( + aot_compile_pool, "arm_parent_death_signal", + lambda: armed.append(True) or True) + request = MintRequest( + function="generate", modules=("m",), family="micro-diffusion", + arm_token="arm2-x", target="/tmp/c.tar.gz", work_root="/tmp", + report="/tmp/r.json", cfg=mint_process.CompileCellSpec(), + posture=posture) + mint_child._install_posture(request) + return nice, armed + + +def test_a_user_machine_mint_child_NICES_ITSELF( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The CPU half of politeness, and the only lever that actually preserves + interactivity — a core reservation does not stop the scheduler putting a + compile on the core the compositor wants; priority does. + + RED before this issue: pgw#1127 §6 recorded **zero** ``os.nice`` calls + anywhere in ``src/``. + """ + nice, _ = _drive_child_posture(monkeypatch, USER_MACHINE) + assert nice.levels == [compile_posture.USER_MACHINE_NICE] == [19] + assert compile_posture.current() == USER_MACHINE + + +def test_a_FLEET_mint_child_never_nices_itself( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The serving pod compiles exactly as fast as it did. A pod's mint already + competes with a tenant whose reserves are held back explicitly; de- + prioritising it on top of that would slow paid work for no gain.""" + nice, armed = _drive_child_posture(monkeypatch, FLEET) + assert nice.levels == [] + assert armed == [] + assert compile_posture.current() == FLEET + + +def test_a_kernel_that_refuses_the_nice_leaves_a_RUDE_mint_not_a_DEAD_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Politeness is not correctness. A container without ``CAP_SYS_NICE`` + must still mint.""" + def _refuse(inc: int) -> int: + raise OSError("not permitted") + + monkeypatch.setattr(os, "nice", _refuse) + monkeypatch.setattr( + aot_compile_pool, "arm_parent_death_signal", lambda: False) + request = MintRequest( + function="generate", modules=("m",), family="f", arm_token="", + target="", work_root="", report="", + cfg=mint_process.CompileCellSpec(), posture=USER_MACHINE) + mint_child._install_posture(request) # must not raise + assert compile_posture.current() == USER_MACHINE + + +def test_the_nice_is_applied_by_the_CHILD_and_never_through_a_preexec_fn( +) -> None: + """A guard, and a deliberate coordination point with pgw#1111. + + ``aot_compile_pool.arm_parent_death_signal``'s own docstring already writes + down why: a ``preexec_fn`` forces ``fork()`` instead of ``posix_spawn()`` + for a process that has live gRPC threads with ``pthread_atfork`` handlers, + and only async-signal-safe work is legal in the forked child. + + Nicing in the mint child also covers strictly MORE. Since pgw#1080 every + production mint is weight-free and ``aot_mint`` forces ``parallel=False`` + for those — so the entry pool spawns no children at all today and the + compile runs in the mint child itself. A ``preexec_fn`` on the pool's spawn + would nice a path that does not currently run; nice is inherited across + ``fork``/``exec``, so one call in the child covers the serial compile, the + entry children when parallelism returns, inductor's compile workers and + every ``cc1plus`` under them. + """ + for name in ("aot_compile_pool.py", "mint_process.py", "mint_child.py"): + tree = ast.parse((SRC / name).read_text()) + passed = { + kw.arg for node in ast.walk(tree) if isinstance(node, ast.Call) + for kw in node.keywords + } + assert "preexec_fn" not in passed, ( + f"{name} must not spawn through preexec_fn — see " + f"arm_parent_death_signal's docstring for the fork/gRPC hazard") + + +# --------------------------------------------------------------------------- +# 3. Memory — a mint that OOMs a desktop is worse than a slow one +# --------------------------------------------------------------------------- + + +def test_a_user_machine_leaves_more_HOST_RAM_alone() -> None: + """``MemAvailable`` counts reclaimable page cache as free, so a pool sized + against it evicts the working set of every application the user has open + before it meets any limit — and on a desktop the OOM killer's most + attractive target is the browser. + + RED: one reserve constant, 4 GiB, sized for a pod where the only thing + being protected is a serving process whose RSS is already excluded. + """ + pod = _width(36, vcpus=64, avail_gib=28, posture=FLEET) + desk = _width(36, vcpus=64, avail_gib=28, posture=USER_MACHINE) + # (28 - 4) / 3 = 8 vs (28 - 8) / 3 = 6 + assert pod.mem_workers == 8 + assert desk.mem_workers == 6 + assert compile_posture.USER_MACHINE_RSS_RESERVE_BYTES == 8 * 1024**3 + + +def test_a_sixteen_gig_desktop_under_load_mints_serially() -> None: + """The case this bound exists for: a laptop with a browser open. 10 GiB + available minus the reserve leaves nothing for a second entry.""" + assert _width( + 36, vcpus=16, avail_gib=10, posture=USER_MACHINE).workers == 1 + + +# --------------------------------------------------------------------------- +# 4. The serving pod does not regress — proven against the OLD formula +# --------------------------------------------------------------------------- + + +def _pre_issue_width( + entries: int, *, vcpus: int, avail: int, free_vram: int, per_device: int, + limit: int = 0, +) -> Dict[str, int]: + """``entry_workers``' arithmetic as it stood BEFORE this issue, restated. + + Deliberately an independent re-implementation and not a call into the code + under test: a non-regression test that asks the new code whether it changed + can only ever answer no. + """ + cpu_workers = max( + 1, (vcpus - aot_compile_pool.SERVING_HEADROOM_CPUS) + // aot_compile_pool.CPUS_PER_ENTRY_WORKER) + per_entry = aot_compile_pool.DEFAULT_ENTRY_PEAK_RSS_BYTES + mem_workers = max( + 1, max(0, avail - aot_compile_pool.ENTRY_RSS_RESERVE_BYTES) + // per_entry) if avail > 0 else 1 + if free_vram <= 0: + device_workers = aot_compile_pool.MAX_ENTRY_WORKERS + elif per_device <= 0: + device_workers = 1 + else: + device_workers = max( + 1, max(0, free_vram - aot_compile_pool.DEVICE_RESERVE_BYTES) + // per_device) + ceiling = ( + min(aot_compile_pool.MAX_ENTRY_WORKERS, limit) if limit > 0 + else aot_compile_pool.MAX_ENTRY_WORKERS) + return { + "cpu_workers": cpu_workers, "mem_workers": mem_workers, + "device_workers": device_workers, "ceiling": ceiling, + "workers": max( + 1, min(cpu_workers, mem_workers, device_workers, ceiling, + entries)), + } + + +@pytest.mark.parametrize( + "entries,vcpus,avail_gib,free_vram_gib,device_gib,limit", [ + (18, 8, 32, 24, 8, 0), # a modest serving pod + (36, 64, 200, 80, 11, 0), # a fat H100 pod + (36, 128, 500, 80, 6, 0), # the widest real pod + (18, 16, 64, 0, 0, 0), # a CPU-only cell (no card at all) + (36, 64, 200, 80, 11, 3), # an operator-forced narrow pool + (4, 4, 8, 24, 8, 0), # a small pod where RAM binds + ]) +def test_a_POD_width_is_arithmetically_unchanged_by_this_issue( + entries: int, vcpus: int, avail_gib: float, free_vram_gib: float, + device_gib: float, limit: int, +) -> None: + """§4.30's constraint the other way round: *"aggressive on a dedicated + serving pod"*. Every bound, over a matrix spanning the real fleet.""" + got = _width( + entries, vcpus=vcpus, avail_gib=avail_gib, posture=FLEET, + free_vram_gib=free_vram_gib, device_gib=device_gib, limit=limit) + want = _pre_issue_width( + entries, vcpus=vcpus, avail=int(avail_gib * 1024**3), + free_vram=int(free_vram_gib * 1024**3), + per_device=int(device_gib * 1024**3), limit=limit) + assert { + "cpu_workers": got.cpu_workers, "mem_workers": got.mem_workers, + "device_workers": got.device_workers, "ceiling": got.ceiling, + "workers": got.workers, + } == want + + +def test_the_DEFAULT_posture_is_the_fleet_one_when_nothing_installed() -> None: + """Nothing on the fleet path installs a posture, so the fallback IS the + fleet policy — the same shape ``worker_goals.current()`` uses, and the + reason no serving code path had to change.""" + assert compile_posture.current() == FLEET + assert _width( + 36, vcpus=32, avail_gib=256, posture=FLEET).cpu_workers \ + == aot_compile_pool.entry_workers( + 36, vcpus=32, available_bytes=256 * 1024**3, free_vram_bytes=0, + device_lock=True, goals=SERVE_ONLY).cpu_workers + + +def test_the_width_row_SAYS_which_posture_chose_K() -> None: + """pgw#842's rule applied to the new bound: an unexplained K is a defect in + itself, and a K held down for a human at a keyboard is a different fact + from a K held down for a tenant.""" + desk = _width(36, vcpus=32, avail_gib=256, posture=USER_MACHINE) + assert desk.facts()["posture"] == "user-machine" + assert desk.facts()["nice"] == 19 + assert "§4.30 user-machine" in desk.reason + assert _width( + 36, vcpus=32, avail_gib=256, posture=FLEET).facts()["posture"] == "fleet" + + +# --------------------------------------------------------------------------- +# 5. Honesty — the user can see the compile and what it costs +# --------------------------------------------------------------------------- + + +def test_the_user_is_told_what_the_compile_COSTS_before_it_starts() -> None: + """*"A silent 20-minute CPU hog is a support ticket."* Four questions the + notice has to answer: what is happening, will it happen again, what is it + costing me, and may I stop it. + + RED: no user-facing surface existed on this path at all — the only signals + were an activity addressed to a hub cozy-local does not have and a + ``logger.info`` inside a subprocess. + """ + facts = aot_compile_pool.CpuFacts( + vcpus=32, basis="caller", os_cpu_count=32, affinity_cpus=32, + quota_cores=-1.0) + said = local_serve.compile_notice( + "micro-diffusion", USER_MACHINE, cpu=facts, + store_root=Path("/home/u/.cache/cozy/compile-cells")) + assert "micro-diffusion" in said + assert "ONCE" in said, "the user must know this is not every run" + assert "/home/u/.cache/cozy/compile-cells" in said, ( + "where the result goes is what makes the promise checkable") + assert "nice 19" in said and "32 cores" in said, ( + "what it is taking, in the units of the machine it is taking it from") + assert "Ctrl-C is safe" in said + + +def test_progress_reaches_the_user_WHILE_the_compile_runs() -> None: + """A cadence, not a burst: the frames arrive at wildly different rates (one + per export, one per compiled entry, then silence through a single long link + step) and a user watching a machine they can no longer type on needs a + steady "still working" line.""" + clock = [0.0] + lines: List[str] = [] + watch = local_serve._Progress( + "micro-diffusion", say=lines.append, interval_s=10.0, + clock=lambda: clock[0]) + + watch(MintFrame(phase="trace_graph", note="exporting")) + assert len(lines) == 1 and "micro-diffusion" in lines[0] + + clock[0] = 3.0 + watch(MintFrame(phase="compile_entries", step=1, total=18)) + assert len(lines) == 1, "throttled: three seconds is not a new line" + + clock[0] = 45.0 + watch(MintFrame(phase="compile_entries", step=7, total=18)) + assert len(lines) == 2 + assert "7/18" in lines[1] and "0m45s" in lines[1], ( + "how far along, and how long it has been — the two numbers a user " + "deciding whether to wait actually needs") + + +def test_the_terminal_watcher_never_displaces_the_HUB_activity() -> None: + """The fleet's own reporting is unchanged: the watcher is an ADDITIONAL + sink, and a fleet mint passes none.""" + phases: List[Tuple[str, int, int]] = [] + notes: List[str] = [] + + class _Act: + def phase(self, p: str, step: int = 0, total: int = 0) -> None: + phases.append((p, step, total)) + + def note(self, n: str) -> None: + notes.append(n) + + seen: List[MintFrame] = [] + frame = MintFrame(phase="compile_entries", step=2, total=18, note="hi") + + mint_delegate._on_frame(_Act())(frame) # the fleet shape + assert phases == [("compile_entries", 2, 18)] and notes == ["hi"] + + mint_delegate._on_frame(_Act(), seen.append)(frame) + assert phases == [("compile_entries", 2, 18)] * 2 + assert seen == [frame] + + +# --------------------------------------------------------------------------- +# 6. Interruptibility — a stopped mint costs nothing and leaves nothing running +# --------------------------------------------------------------------------- + + +def test_a_user_machine_mint_child_DIES_WITH_ITS_PARENT( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``mint_process`` spawns the child with ``start_new_session=True``, so it + holds its OWN session and a terminal's Ctrl-C or SIGHUP never reaches it. + + On a pod that is correct — the parent reaps the group deliberately when it + abandons a mint. On a desktop the parent is a CLI a human closes, and + without ``PR_SET_PDEATHSIG`` a closed terminal leaves a full-speed compile + tree running with nobody left to reap it. That is precisely the "my machine + is at a crawl and I don't know why" ticket this issue exists to prevent. + + RED: ``arm_parent_death_signal`` had exactly one caller, the entry-pool + child, and the mint child armed nothing. + """ + _, armed = _drive_child_posture(monkeypatch, USER_MACHINE) + assert armed == [True] + + +def test_a_stopped_local_mint_does_not_WASTE_what_it_finished() -> None: + """The claim the notice makes out loud ("Ctrl-C is safe"), pinned. + + ``aot_resume``'s bank is keyed by the pending's arm token and sited OUTSIDE + the per-attempt workdir on purpose — ``abandon_self_mint`` rmtree's + ``mint_root``, and abandonment is how a killed mint ends. A bank inside the + workdir would be deleted on its way out of the one case it exists for, and + the notice would become a lie. + """ + workdir = Path("/tmp/pgw1137/child-1") + request = mint_delegate.build_request( + mint_delegate.MintTask( + pending=_Pending(), pipe=None, function="generate", + modules=("m",), posture=USER_MACHINE), + workdir=workdir, cap_bytes=0) + bank = Path(request.resume) + assert str(bank), "a local mint must bank its finished entries" + assert workdir not in bank.parents and bank != workdir, ( + "the resume bank must outlive the attempt AND the pending, or a " + "cancelled compile throws away every entry it had already finished") + + +def test_the_local_notice_and_the_local_posture_cannot_drift( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """One posture object feeds both the policy and the sentence describing it, + so the numbers a user is shown are the numbers the pool used.""" + facts = aot_compile_pool.CpuFacts( + vcpus=8, basis="caller", os_cpu_count=8, affinity_cpus=8, + quota_cores=-1.0) + said = local_serve.compile_notice("f", USER_MACHINE, cpu=facts, + store_root=Path("/r")) + width = _width(36, vcpus=8, avail_gib=256, posture=USER_MACHINE) + assert f"{width.ceiling} parallel worker(s)" in said + assert f"nice {width.posture.nice_level()}" in said + + +def _module_names(path: Path) -> List[str]: + tree = ast.parse(path.read_text()) + out: List[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + out.append(node.module or "") + out += [a.name for a in node.names] + elif isinstance(node, ast.Import): + out += [a.name for a in node.names] + return out + + +def test_the_politeness_wiring_adds_no_transport_to_the_local_serve_entry( +) -> None: + """pgw#1127 §4's fence, re-asserted against THIS issue's imports: the local + serve entry gained ``aot_compile_pool``, ``compile_posture`` and + ``local_cell_store``, and none of them may drag a publisher in.""" + names = _module_names(SRC / "local_serve.py") + assert "compile_posture" in names and "aot_compile_pool" in names + for banned in ("cell_publish", "cas_client", "httpx", "requests", + "aiohttp"): + assert banned not in names + + +def test_the_posture_module_holds_POLICY_and_nothing_else() -> None: + """Every politeness term lives in one place, so a reader can price the + whole trade without walking four modules — and so the notice, the pool and + the child cannot each grow their own idea of what polite means.""" + posture = CompilePosture(user_machine=True) + assert posture.nice_level() == 19 + assert posture.entry_ceiling(8) == 4 + assert posture.entry_ceiling(2) == 2, "never widens" + assert posture.rss_reserve_bytes(4 * 1024**3) == 8 * 1024**3 + assert posture.rss_reserve_bytes(16 * 1024**3) == 16 * 1024**3, \ + "never shrinks" + assert posture.cpu_budget_cores(32, headroom=2) == 16 + assert posture.cpu_budget_cores(4, headroom=2) == 2 + + fleet = CompilePosture() + assert fleet.nice_level() == 0 + assert fleet.entry_ceiling(8) == 8 + assert fleet.rss_reserve_bytes(4 * 1024**3) == 4 * 1024**3 + assert fleet.cpu_budget_cores(32, headroom=2) == 30 + + +_ = Optional, Iterator, Any